View: the pipelines heard kept for the pages is the setting pipelines-heard
The view kept the last change of at most 1024 pipelines for its pages, a cap written into the code. A cap is a setting with a default (principle 11): the manifest gives pipelines-heard, 1024 by default, written into a file of the state directory the process reads at its start. The cap is kept, not removed: without it the pipelines heard grow without end over the view's life. Issue 1983, review round 1 finding 3. Builds: design 47 H2 H-A-467 (1)
This commit is contained in:
@@ -325,7 +325,7 @@ func TestThePipelinesCommands(t *testing.T) {
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
m.SetPipelines(view.NewPipelines(call, nil, nil))
|
||||
m.SetPipelines(view.NewPipelines(call, nil, 0, nil))
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "p")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"maps"
|
||||
"mesh-delivery-view/internal/view"
|
||||
@@ -22,6 +23,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
@@ -193,7 +195,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// The pipelines page reads the seat mesh-pipeline-runner's list and show, and reads again on the runner's events heard on
|
||||
// the module's one pull, never on a timer (design 47 H2 H-A-467); the watched branches' lists link each delivery.
|
||||
pipes := view.NewPipelines(calls.Call, branches.Followed, logger.Printf)
|
||||
heard, err := numberFile(getenv("MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"), view.DefaultPipelinesHeard)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the setting pipelines-heard: %w", err)
|
||||
}
|
||||
pipes := view.NewPipelines(calls.Call, branches.Followed, heard, logger.Printf)
|
||||
model.SetPipelines(pipes)
|
||||
if err := hearPipelines(hub, pipes, logger.Printf); err != nil {
|
||||
return fmt.Errorf("the runner's events are not handled: %w", err)
|
||||
@@ -544,3 +550,27 @@ func (c *catalogModules) apply(module string, entry meshcatalog.CatalogEntry, li
|
||||
e.History = true
|
||||
c.model.Apply(e)
|
||||
}
|
||||
|
||||
// numberFile reads a setting the node-engine writes into a file of the state directory: def when path is empty, the file
|
||||
// missing or empty; an error when it holds no number of at least one.
|
||||
func numberFile(path string, def int) (int, error) {
|
||||
if path == "" {
|
||||
return def, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return def, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return def, nil
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a number of at least one", path, v)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -17,6 +19,37 @@ func TestRunNeedsItsCredential(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The setting pipelines-heard is read from the file the node-engine writes, its default where none is written (principle 11).
|
||||
func TestNumberFileReadsASetting(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, v string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(v), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
for _, r := range []struct {
|
||||
name, path string
|
||||
want int
|
||||
fails bool
|
||||
}{
|
||||
{"no file named is the default", "", 1024, false},
|
||||
{"a file not written is the default", filepath.Join(dir, "none"), 1024, false},
|
||||
{"an empty file is the default", write("empty", "\n"), 1024, false},
|
||||
{"a number is read", write("seven", "7\n"), 7, false},
|
||||
{"zero is refused", write("zero", "0"), 0, true},
|
||||
{"words are refused", write("words", "many"), 0, true},
|
||||
} {
|
||||
t.Run(r.name, func(t *testing.T) {
|
||||
n, err := numberFile(r.path, 1024)
|
||||
if (err != nil) != r.fails || n != r.want {
|
||||
t.Errorf("numberFile = %d, %v; want %d, failing %v", n, err, r.want, r.fails)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// rawAsk is the router's ask as the JSON a gate's or an assigner's value carries (mesh-sdk keeps it raw, ADR 0627 section 4).
|
||||
func rawAsk(t *testing.T, a oc.Ask) json.RawMessage {
|
||||
t.Helper()
|
||||
|
||||
@@ -23,6 +23,9 @@ const (
|
||||
const (
|
||||
DefaultPipelinesBranch = "main"
|
||||
DefaultPipelinesLimit = 20
|
||||
// DefaultPipelinesHeard is the default of the setting pipelines-heard: how many pipelines' last changes the view keeps
|
||||
// for its pages before it forgets them all (principle 11).
|
||||
DefaultPipelinesHeard = 1024
|
||||
)
|
||||
|
||||
// PipelineEvents are the runner's events the pipelines page refreshes on (design 47 H2 H-A-467 (1)), never a timer.
|
||||
@@ -59,16 +62,21 @@ type Pipelines struct {
|
||||
subs map[chan string]struct{}
|
||||
version uint64
|
||||
heard map[string]uint64 // the version of the last change of each pipeline heard
|
||||
most int // the pipelines heard kept before heard is forgotten whole: the setting pipelines-heard
|
||||
}
|
||||
|
||||
// NewPipelines is a reader of the pipelines through call; branches, when given, names the watched branches whose list is
|
||||
// followed beside the default branch's, for the deliveries' links (design 47 H2 H-A-467 (3)).
|
||||
func NewPipelines(call func(key string, body any) (json.RawMessage, error), branches func() []string, logf func(string, ...any)) *Pipelines {
|
||||
// followed beside the default branch's, for the deliveries' links (design 47 H2 H-A-467 (3)); heard is the setting
|
||||
// pipelines-heard, its default below one.
|
||||
func NewPipelines(call func(key string, body any) (json.RawMessage, error), branches func() []string, heard int, logf func(string, ...any)) *Pipelines {
|
||||
if logf == nil {
|
||||
logf = func(string, ...any) {}
|
||||
}
|
||||
if heard < 1 {
|
||||
heard = DefaultPipelinesHeard
|
||||
}
|
||||
return &Pipelines{call: call, branches: branches, logf: logf, dirty: make(chan struct{}, 1),
|
||||
lists: map[string]PipelinesPage{}, subs: map[chan string]struct{}{}, heard: map[string]uint64{}}
|
||||
lists: map[string]PipelinesPage{}, subs: map[chan string]struct{}{}, heard: map[string]uint64{}, most: heard}
|
||||
}
|
||||
|
||||
// List asks the seat's verb list for one branch's newest limit pipelines in its short form; a limit below one is the
|
||||
@@ -231,7 +239,7 @@ func (p *Pipelines) notify(what string) {
|
||||
defer p.mu.Unlock()
|
||||
p.version++
|
||||
if what != "" {
|
||||
if len(p.heard) > 1024 {
|
||||
if _, known := p.heard[what]; !known && len(p.heard) >= p.most {
|
||||
clear(p.heard)
|
||||
}
|
||||
p.heard[what] = p.version
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestThePipelinesPage(t *testing.T) {
|
||||
Steps: map[string]int64{"passed": 5, "failed": 1, "could not run": 1, "waiting for a build node": 2}})
|
||||
runner.setRows(testBranch, seat.PipelineRow{Pipeline: "release.8", Source: 4, Revision: 8, Outcome: seat.PipelineOutcomePassed},
|
||||
seat.PipelineRow{Pipeline: "release.7", Source: 4, Revision: 7, Outcome: seat.PipelineOutcomeSuperseded})
|
||||
p := NewPipelines(runner.call, func() []string { return []string{testBranch} }, nil)
|
||||
p := NewPipelines(runner.call, func() []string { return []string{testBranch} }, 0, nil)
|
||||
m := New(testView)
|
||||
fill(m, sourceE(t, 4))
|
||||
m.SetPipelines(p)
|
||||
@@ -237,6 +237,22 @@ func TestThePipelinesPage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the pipelines heard are kept up to the setting pipelines-heard", func(t *testing.T) {
|
||||
few := NewPipelines(runner.call, nil, 2, nil)
|
||||
for _, k := range []string{"main.1", "main.2", "main.2", "main.3"} {
|
||||
few.notify(k)
|
||||
}
|
||||
few.mu.Lock()
|
||||
n, last := len(few.heard), few.heard["main.3"]
|
||||
few.mu.Unlock()
|
||||
if n > 2 || last == 0 {
|
||||
t.Errorf("heard holds %d pipelines (main.3 at %d), want at most the setting's 2, the newest kept", n, last)
|
||||
}
|
||||
if d := NewPipelines(runner.call, nil, 0, nil); d.most != DefaultPipelinesHeard {
|
||||
t.Errorf("the default keeps %d, want %d", d.most, DefaultPipelinesHeard)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a list not read is shown unreadable and followed on, read again at the next event", func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
away := true // the runner not answering yet, as the bus says when no holder listens
|
||||
@@ -248,7 +264,7 @@ func TestThePipelinesPage(t *testing.T) {
|
||||
}
|
||||
return runner.call(key, body)
|
||||
}
|
||||
up := NewPipelines(call, nil, nil)
|
||||
up := NewPipelines(call, nil, 0, nil)
|
||||
upCtx, upCancel := context.WithCancel(ctx)
|
||||
defer upCancel()
|
||||
go up.Run(upCtx)
|
||||
|
||||
@@ -71,26 +71,33 @@ type claim struct {
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Packages map[string]string `json:"packages"`
|
||||
Claims []claim `json:"claims"`
|
||||
Class string `json:"class"`
|
||||
State []state `json:"state"`
|
||||
Module string `json:"module"`
|
||||
Moved bool `json:"moved"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Invokes []invoke `json:"invokes"`
|
||||
Uses []string `json:"uses"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Streams []stream `json:"streams"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Requires []require `json:"requires"`
|
||||
Items []item `json:"items"`
|
||||
Packages map[string]string `json:"packages"`
|
||||
Claims []claim `json:"claims"`
|
||||
Class string `json:"class"`
|
||||
State []state `json:"state"`
|
||||
Module string `json:"module"`
|
||||
Moved bool `json:"moved"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Invokes []invoke `json:"invokes"`
|
||||
Uses []string `json:"uses"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Streams []stream `json:"streams"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Trusted bool `json:"trusted"`
|
||||
Minimum float64 `json:"minimum"`
|
||||
Default any `json:"default"`
|
||||
} `json:"config"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Requires []require `json:"requires"`
|
||||
Items []item `json:"items"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
@@ -387,6 +394,17 @@ func TestManifestResources(t *testing.T) { // 1134/AC7
|
||||
if env["MESH_BROKER_FILE"] != "${dir:state}/broker" || m.OwnSecrets["broker"] != "${dir:state}/broker" {
|
||||
t.Errorf("the process reads %v, the own-secret is written at %v", env["MESH_BROKER_FILE"], m.OwnSecrets["broker"])
|
||||
}
|
||||
// The setting pipelines-heard reaches the process as a file of its state directory, read at its start (principle 11).
|
||||
if c, ok := m.Config["pipelines-heard"]; !ok || c.Default != float64(view.DefaultPipelinesHeard) || c.Minimum != 1 || c.Type != "number" || len(m.Config) != 1 {
|
||||
t.Errorf("config = %+v, want pipelines-heard, a number of at least 1, %d by default", m.Config, view.DefaultPipelinesHeard)
|
||||
}
|
||||
var heardFile bool
|
||||
for _, r := range byType["file"] {
|
||||
heardFile = heardFile || (r["id"] == "pipelines-heard" && r["path"] == "${dir:state}/pipelines-heard" && r["content"] == "${config:pipelines-heard}" && r["owner"] == user && r["mode"] == "0600")
|
||||
}
|
||||
if !heardFile || env["MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"] != "${dir:state}/pipelines-heard" {
|
||||
t.Errorf("the setting pipelines-heard is written to its file %v, the process reads %v", heardFile, env["MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"])
|
||||
}
|
||||
if h, _ := p["health"].(map[string]any); h["kind"] != "unit" {
|
||||
t.Errorf("health = %v", p["health"])
|
||||
}
|
||||
|
||||
@@ -154,6 +154,15 @@
|
||||
"seat:mesh-vault.signers",
|
||||
"node-engine.have"
|
||||
],
|
||||
"config": {
|
||||
"pipelines-heard": {
|
||||
"type": "number",
|
||||
"description": "How many pipelines' last changes the view keeps for its pipelines pages before it forgets them all, read at its start (design 47 H2 H-A-467; principle 11).",
|
||||
"trusted": false,
|
||||
"minimum": 1,
|
||||
"default": 1024
|
||||
}
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
@@ -191,13 +200,22 @@
|
||||
"./mesh-delivery-view"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE": "${dir:state}/pipelines-heard"
|
||||
},
|
||||
"user": "mesh-delivery-view",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "pipelines-heard",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/pipelines-heard",
|
||||
"owner": "mesh-delivery-view",
|
||||
"mode": "0600",
|
||||
"content": "${config:pipelines-heard}"
|
||||
},
|
||||
{
|
||||
"id": "command-program",
|
||||
"type": "archive",
|
||||
|
||||
Reference in New Issue
Block a user