From d151033dae555ab79a8c6e0b9d0e59c6e030d09f Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 11:45:32 +0100 Subject: [PATCH] View: the pipelines heard kept for the pages is the setting pipelines-heard The view kept the last change of at most 1024 pipelines for its pages, a cap written into the code. A cap is a setting with a default (principle 11): the manifest gives pipelines-heard, 1024 by default, written into a file of the state directory the process reads at its start. The cap is kept, not removed: without it the pipelines heard grow without end over the view's life. Issue 1983, review round 1 finding 3. Builds: design 47 H2 H-A-467 (1) --- .../cmd/mesh-delivery-cli/main_test.go | 2 +- .../cmd/mesh-delivery-view/main.go | 32 +++++++++- .../cmd/mesh-delivery-view/main_test.go | 33 +++++++++++ .../internal/view/pipelines.go | 16 +++-- .../internal/view/pipelines_test.go | 20 ++++++- modules/mesh-delivery-view/manifest_test.go | 58 ++++++++++++------- modules/mesh-delivery-view/module.json | 20 ++++++- 7 files changed, 152 insertions(+), 29 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go index 2f3634919..4d4f61386 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main_test.go @@ -325,7 +325,7 @@ func TestThePipelinesCommands(t *testing.T) { } return nil, nil } - m.SetPipelines(view.NewPipelines(call, nil, nil)) + m.SetPipelines(view.NewPipelines(call, nil, 0, nil)) dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "p") if err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 1303a19f2..ca1ec3863 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "io" + "io/fs" "log" "maps" "mesh-delivery-view/internal/view" @@ -22,6 +23,7 @@ import ( "os" "os/signal" "slices" + "strconv" "strings" "sync" "syscall" @@ -193,7 +195,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // The pipelines page reads the seat mesh-pipeline-runner's list and show, and reads again on the runner's events heard on // the module's one pull, never on a timer (design 47 H2 H-A-467); the watched branches' lists link each delivery. - pipes := view.NewPipelines(calls.Call, branches.Followed, logger.Printf) + heard, err := numberFile(getenv("MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"), view.DefaultPipelinesHeard) + if err != nil { + return fmt.Errorf("the setting pipelines-heard: %w", err) + } + pipes := view.NewPipelines(calls.Call, branches.Followed, heard, logger.Printf) model.SetPipelines(pipes) if err := hearPipelines(hub, pipes, logger.Printf); err != nil { return fmt.Errorf("the runner's events are not handled: %w", err) @@ -544,3 +550,27 @@ func (c *catalogModules) apply(module string, entry meshcatalog.CatalogEntry, li e.History = true c.model.Apply(e) } + +// numberFile reads a setting the node-engine writes into a file of the state directory: def when path is empty, the file +// missing or empty; an error when it holds no number of at least one. +func numberFile(path string, def int) (int, error) { + if path == "" { + return def, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return def, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return def, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return 0, fmt.Errorf("%s holds %q, which is not a number of at least one", path, v) + } + return n, nil +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main_test.go index f045cac0b..fb6ff3bb7 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "io" + "os" + "path/filepath" "strings" "testing" @@ -17,6 +19,37 @@ func TestRunNeedsItsCredential(t *testing.T) { } } +// The setting pipelines-heard is read from the file the node-engine writes, its default where none is written (principle 11). +func TestNumberFileReadsASetting(t *testing.T) { + dir := t.TempDir() + write := func(name, v string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(v), 0o600); err != nil { + t.Fatal(err) + } + return p + } + for _, r := range []struct { + name, path string + want int + fails bool + }{ + {"no file named is the default", "", 1024, false}, + {"a file not written is the default", filepath.Join(dir, "none"), 1024, false}, + {"an empty file is the default", write("empty", "\n"), 1024, false}, + {"a number is read", write("seven", "7\n"), 7, false}, + {"zero is refused", write("zero", "0"), 0, true}, + {"words are refused", write("words", "many"), 0, true}, + } { + t.Run(r.name, func(t *testing.T) { + n, err := numberFile(r.path, 1024) + if (err != nil) != r.fails || n != r.want { + t.Errorf("numberFile = %d, %v; want %d, failing %v", n, err, r.want, r.fails) + } + }) + } +} + // rawAsk is the router's ask as the JSON a gate's or an assigner's value carries (mesh-sdk keeps it raw, ADR 0627 section 4). func rawAsk(t *testing.T, a oc.Ask) json.RawMessage { t.Helper() diff --git a/modules/mesh-delivery-view/internal/view/pipelines.go b/modules/mesh-delivery-view/internal/view/pipelines.go index e3867499c..0428d126f 100644 --- a/modules/mesh-delivery-view/internal/view/pipelines.go +++ b/modules/mesh-delivery-view/internal/view/pipelines.go @@ -23,6 +23,9 @@ const ( const ( DefaultPipelinesBranch = "main" DefaultPipelinesLimit = 20 + // DefaultPipelinesHeard is the default of the setting pipelines-heard: how many pipelines' last changes the view keeps + // for its pages before it forgets them all (principle 11). + DefaultPipelinesHeard = 1024 ) // PipelineEvents are the runner's events the pipelines page refreshes on (design 47 H2 H-A-467 (1)), never a timer. @@ -59,16 +62,21 @@ type Pipelines struct { subs map[chan string]struct{} version uint64 heard map[string]uint64 // the version of the last change of each pipeline heard + most int // the pipelines heard kept before heard is forgotten whole: the setting pipelines-heard } // NewPipelines is a reader of the pipelines through call; branches, when given, names the watched branches whose list is -// followed beside the default branch's, for the deliveries' links (design 47 H2 H-A-467 (3)). -func NewPipelines(call func(key string, body any) (json.RawMessage, error), branches func() []string, logf func(string, ...any)) *Pipelines { +// followed beside the default branch's, for the deliveries' links (design 47 H2 H-A-467 (3)); heard is the setting +// pipelines-heard, its default below one. +func NewPipelines(call func(key string, body any) (json.RawMessage, error), branches func() []string, heard int, logf func(string, ...any)) *Pipelines { if logf == nil { logf = func(string, ...any) {} } + if heard < 1 { + heard = DefaultPipelinesHeard + } return &Pipelines{call: call, branches: branches, logf: logf, dirty: make(chan struct{}, 1), - lists: map[string]PipelinesPage{}, subs: map[chan string]struct{}{}, heard: map[string]uint64{}} + lists: map[string]PipelinesPage{}, subs: map[chan string]struct{}{}, heard: map[string]uint64{}, most: heard} } // List asks the seat's verb list for one branch's newest limit pipelines in its short form; a limit below one is the @@ -231,7 +239,7 @@ func (p *Pipelines) notify(what string) { defer p.mu.Unlock() p.version++ if what != "" { - if len(p.heard) > 1024 { + if _, known := p.heard[what]; !known && len(p.heard) >= p.most { clear(p.heard) } p.heard[what] = p.version diff --git a/modules/mesh-delivery-view/internal/view/pipelines_test.go b/modules/mesh-delivery-view/internal/view/pipelines_test.go index a1220cf2b..0f1ae322f 100644 --- a/modules/mesh-delivery-view/internal/view/pipelines_test.go +++ b/modules/mesh-delivery-view/internal/view/pipelines_test.go @@ -81,7 +81,7 @@ func TestThePipelinesPage(t *testing.T) { Steps: map[string]int64{"passed": 5, "failed": 1, "could not run": 1, "waiting for a build node": 2}}) runner.setRows(testBranch, seat.PipelineRow{Pipeline: "release.8", Source: 4, Revision: 8, Outcome: seat.PipelineOutcomePassed}, seat.PipelineRow{Pipeline: "release.7", Source: 4, Revision: 7, Outcome: seat.PipelineOutcomeSuperseded}) - p := NewPipelines(runner.call, func() []string { return []string{testBranch} }, nil) + p := NewPipelines(runner.call, func() []string { return []string{testBranch} }, 0, nil) m := New(testView) fill(m, sourceE(t, 4)) m.SetPipelines(p) @@ -237,6 +237,22 @@ func TestThePipelinesPage(t *testing.T) { } }) + t.Run("the pipelines heard are kept up to the setting pipelines-heard", func(t *testing.T) { + few := NewPipelines(runner.call, nil, 2, nil) + for _, k := range []string{"main.1", "main.2", "main.2", "main.3"} { + few.notify(k) + } + few.mu.Lock() + n, last := len(few.heard), few.heard["main.3"] + few.mu.Unlock() + if n > 2 || last == 0 { + t.Errorf("heard holds %d pipelines (main.3 at %d), want at most the setting's 2, the newest kept", n, last) + } + if d := NewPipelines(runner.call, nil, 0, nil); d.most != DefaultPipelinesHeard { + t.Errorf("the default keeps %d, want %d", d.most, DefaultPipelinesHeard) + } + }) + t.Run("a list not read is shown unreadable and followed on, read again at the next event", func(t *testing.T) { var mu sync.Mutex away := true // the runner not answering yet, as the bus says when no holder listens @@ -248,7 +264,7 @@ func TestThePipelinesPage(t *testing.T) { } return runner.call(key, body) } - up := NewPipelines(call, nil, nil) + up := NewPipelines(call, nil, 0, nil) upCtx, upCancel := context.WithCancel(ctx) defer upCancel() go up.Run(upCtx) diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 63ee5a3b2..ea0ebe9db 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -71,26 +71,33 @@ type claim struct { } type manifest struct { - Packages map[string]string `json:"packages"` - Claims []claim `json:"claims"` - Class string `json:"class"` - State []state `json:"state"` - Module string `json:"module"` - Moved bool `json:"moved"` - Version string `json:"version"` - Slug string `json:"slug"` - Reads []string `json:"reads"` - Consumes []string `json:"consumes"` - Invokes []invoke `json:"invokes"` - Uses []string `json:"uses"` - Tools []string `json:"tools"` - Emits []string `json:"emits"` - Streams []stream `json:"streams"` - OwnSecrets map[string]string `json:"own-secrets"` - SecretsOwner string `json:"secrets-owner"` - Resources []map[string]any `json:"resources"` - Requires []require `json:"requires"` - Items []item `json:"items"` + Packages map[string]string `json:"packages"` + Claims []claim `json:"claims"` + Class string `json:"class"` + State []state `json:"state"` + Module string `json:"module"` + Moved bool `json:"moved"` + Version string `json:"version"` + Slug string `json:"slug"` + Reads []string `json:"reads"` + Consumes []string `json:"consumes"` + Invokes []invoke `json:"invokes"` + Uses []string `json:"uses"` + Tools []string `json:"tools"` + Emits []string `json:"emits"` + Streams []stream `json:"streams"` + OwnSecrets map[string]string `json:"own-secrets"` + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Trusted bool `json:"trusted"` + Minimum float64 `json:"minimum"` + Default any `json:"default"` + } `json:"config"` + SecretsOwner string `json:"secrets-owner"` + Resources []map[string]any `json:"resources"` + Requires []require `json:"requires"` + Items []item `json:"items"` Build struct { Artifacts []artifact `json:"artifacts"` } `json:"build"` @@ -387,6 +394,17 @@ func TestManifestResources(t *testing.T) { // 1134/AC7 if env["MESH_BROKER_FILE"] != "${dir:state}/broker" || m.OwnSecrets["broker"] != "${dir:state}/broker" { t.Errorf("the process reads %v, the own-secret is written at %v", env["MESH_BROKER_FILE"], m.OwnSecrets["broker"]) } + // The setting pipelines-heard reaches the process as a file of its state directory, read at its start (principle 11). + if c, ok := m.Config["pipelines-heard"]; !ok || c.Default != float64(view.DefaultPipelinesHeard) || c.Minimum != 1 || c.Type != "number" || len(m.Config) != 1 { + t.Errorf("config = %+v, want pipelines-heard, a number of at least 1, %d by default", m.Config, view.DefaultPipelinesHeard) + } + var heardFile bool + for _, r := range byType["file"] { + heardFile = heardFile || (r["id"] == "pipelines-heard" && r["path"] == "${dir:state}/pipelines-heard" && r["content"] == "${config:pipelines-heard}" && r["owner"] == user && r["mode"] == "0600") + } + if !heardFile || env["MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"] != "${dir:state}/pipelines-heard" { + t.Errorf("the setting pipelines-heard is written to its file %v, the process reads %v", heardFile, env["MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE"]) + } if h, _ := p["health"].(map[string]any); h["kind"] != "unit" { t.Errorf("health = %v", p["health"]) } diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 51ee46cf0..d7c077085 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -154,6 +154,15 @@ "seat:mesh-vault.signers", "node-engine.have" ], + "config": { + "pipelines-heard": { + "type": "number", + "description": "How many pipelines' last changes the view keeps for its pipelines pages before it forgets them all, read at its start (design 47 H2 H-A-467; principle 11).", + "trusted": false, + "minimum": 1, + "default": 1024 + } + }, "own-secrets": { "broker": "${dir:state}/broker" }, @@ -191,13 +200,22 @@ "./mesh-delivery-view" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_DELIVERY_VIEW_PIPELINES_HEARD_FILE": "${dir:state}/pipelines-heard" }, "user": "mesh-delivery-view", "health": { "kind": "unit" } }, + { + "id": "pipelines-heard", + "type": "file", + "path": "${dir:state}/pipelines-heard", + "owner": "mesh-delivery-view", + "mode": "0600", + "content": "${config:pipelines-heard}" + }, { "id": "command-program", "type": "archive",