From 52056010fe643f80613a2c3ba911cbc8afa53f0e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 01:30:50 +0100 Subject: [PATCH 01/52] A module composed as what runs of it serves its tools as they run, so the runtime no longer drops it (issue 1126) After the take-over of placement on 2026-10-29, shanks's runtime stopped serving the folded systemd-resolved (the node-resolver seat's verbs answer "nothing serves") and novox's the folded nats. Neither has a want: each runs inside its node's own composition at a build of the old form (Scope.InOwn), and is composed as Carried of the indexer's manifest. That manifest is the catalogue's, which no build resolved, so it has no bundles (catalogue/build.go: Bundles are set only by a build's resolution), and the runtime's MESH_TOOL_MODULES and MESH_TOOL_ENV are made from bundles (catalogue/runtime.go). The controller composed them from the resolved manifest its store kept. The kind: a module composed as what runs of it (Stub, Carried: held, or folded at an old build) contributes to the others from its manifest, not from what runs. For a held module with a new build it is the other way round: its new bundles are served, which were never delivered. - The quick fix: read the old-form build's resolved manifest for a folded module. One instance; a held module still hands its new bundles. - The thorough fix, chosen: compose.ToolsRunning reads what the node's runtime serves as it runs (its process's MESH_TOOL_MODULES and MESH_TOOL_ENV, each bundle's archive), frombus gives it as Node.RunsTools, and every stubbed module of the machine's runtime (held or carried) contributes those bundles, whatever its manifest says. Tests: two rows (folded, held) added to TestCompositionsAreTheDeclarationSplitByModule; no new test function. --- go/compose/composed.go | 89 +++++++++++++++++++++++++++++++++++ go/compose/composed_test.go | 75 +++++++++++++++++++++++++++++ go/compose/frombus/frombus.go | 2 + go/compose/input.go | 6 +++ 4 files changed, 172 insertions(+) diff --git a/go/compose/composed.go b/go/compose/composed.go index b5a7d7a..66fc29c 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -130,6 +130,22 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli n.Kept = kept } } + // A module composed as what runs of it contributes to the runtime what the runtime serves of it as it runs + // (Node.RunsTools); one of its own account is served by a runtime of its own, from its manifest as before. + for i := range reading.Nodes { + n := &reading.Nodes[i] + if n.Name != node || n.RunsTools == nil { + continue + } + kept := make(map[string]catalogue.Manifest, len(n.Kept)) + for m, man := range n.Kept { + if man.Stubbed && man.RunsAs == "" { + man.Bundles = n.RunsTools[m] + } + kept[m] = man + } + n.Kept = kept + } isCarried := func(module, id string) bool { if module == "" { module, _ = declaration.ModuleGuess(id) @@ -293,6 +309,79 @@ func RunsAnyOf(runs *delivery.Composed, module string) bool { return false } +// ToolsRunning is, by module, the tools bundles the node's runtime serves as it runs (Node.RunsTools): read from the +// runtime's process in running (catalogue.RuntimeToolModules, catalogue.RuntimeToolEnv, the mesh's own words left out) +// and each bundle's archive there; nil when running holds no runtime process. +func ToolsRunning(running []delivery.Composed) map[string][]catalogue.Bundle { + runtimeID := catalogue.RuntimeModule + "." + catalogue.RuntimeProcessID() + type resource struct { + ID string `json:"id"` + Type string `json:"type"` + Path string `json:"path"` + Source string `json:"source"` + Digest string `json:"digest"` + Env map[string]string `json:"env"` + } + var env map[string]string + archives := map[string]resource{} + for _, c := range running { + for _, raw := range c.Resources { + var r resource + if json.Unmarshal(raw, &r) != nil { + continue + } + switch { + case r.Type == "process" && r.ID == runtimeID: + env = r.Env + case r.Type == "archive" && strings.HasPrefix(r.Path, catalogue.BundleRoot+"/"): + archives[r.Path] = r + } + } + } + if env == nil { + return nil + } + var words map[string]map[string]string + _ = json.Unmarshal([]byte(env[catalogue.RuntimeToolEnv]), &words) + out := map[string][]catalogue.Bundle{} + for _, entry := range strings.Split(env[catalogue.RuntimeToolModules], ",") { + module, path, ok := strings.Cut(entry, "=") + if !ok { + continue + } + name, load, ok := strings.Cut(strings.TrimPrefix(path, catalogue.BundleRoot+"/"+module+"/"), "/") + if !ok || !strings.HasPrefix(path, catalogue.BundleRoot+"/"+module+"/") { + continue + } + var own map[string]string + for w, v := range words[module] { + if w == catalogue.FilesOwnedPaths || w == catalogue.ReadsSecretPlaces { + continue // the mesh's to say, composed again + } + if own == nil { + own = map[string]string{} + } + own[w] = v + } + bundles := out[module] + i := slices.IndexFunc(bundles, func(b catalogue.Bundle) bool { return b.Name == name }) + if i < 0 { + a, delivered := archives[catalogue.BundlePath(module, name)] + if !delivered { + continue // no archive of it runs: nothing to load + } + bundles = append(bundles, catalogue.Bundle{Name: name, Source: catalogue.Recorded(a.Source), Digest: a.Digest, Env: own}) + i = len(bundles) - 1 + } + bundles[i].Loads = append(bundles[i].Loads, load) + out[module] = bundles + } + for _, bundles := range out { + sort.Slice(bundles, func(i, j int) bool { return bundles[i].Name < bundles[j].Name }) + } + return out +} + // NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot // compose such a module folded into its own composition: the build of the old form it runs is held by the controller // alone (issue 1126). diff --git a/go/compose/composed_test.go b/go/compose/composed_test.go index 589f156..eb37442 100644 --- a/go/compose/composed_test.go +++ b/go/compose/composed_test.go @@ -8,6 +8,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" ) func fpFor(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } @@ -96,6 +97,80 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) { if _, _, err := Compositions(in, "nowhere", fps, Foreseeing); err == nil { t.Errorf("a node the mesh does not have was composed") } + + // Issue 1126, 2026-10-29: a module composed as what runs of it (folded at a build of the old form, or held) still + // contributes to the runtime what the runtime serves of it as it runs (Node.RunsTools): its manifest at the + // indexer's commit has no bundles (shanks's runtime lost the folded systemd-resolved, novox's the folded nats), and a + // held module's new build's bundles are not delivered. + dg := func(c byte) string { return "sha256:" + strings.Repeat(string(c), 64) } + runtime := catalogue.Manifest{Module: catalogue.RuntimeModule, Version: "1", + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/node-tools/broker"}}, + Bundles: []catalogue.Bundle{{Name: "runtime", Source: catalogue.ArtifactStoreScheme + "node-tools/runtime/blobs/" + dg('a'), + Digest: dg('a'), Language: "go", Binary: "node-tools"}}} + store := catalogue.Manifest{Module: "artifacts", Version: "1", + Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": 5100}}} + lamp := catalogue.Manifest{Module: "lamp", Version: "1", Tools: []string{"on"}, + Resources: []map[string]any{{"id": "state", "type": "directory", "path": "/var/lib/lamp", "mode": "0700"}}, + Build: &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "tools", Kind: catalogue.ArtifactBundle, + Language: "go", System: "arch", From: "cmd/lamp-tools"}}}} + newLamp, err := lamp.Resolve([]catalogue.Built{{Name: "tools", Kind: catalogue.ArtifactBundle, + Reference: catalogue.ArtifactStoreScheme + "lamp/tools/blobs/" + dg('c'), Digest: dg('c')}}) + if err != nil { + t.Fatal(err) + } + served := "lamp=" + catalogue.BundlePath("lamp", "tools") + "/lamp-tools" + runsTools := []delivery.Composed{{Node: "anchor", Module: catalogue.RuntimeModule, Resources: []json.RawMessage{json.RawMessage( + `{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","env":{"MESH_TOOL_MODULES":"` + + served + `","MESH_TOOL_ENV":"{\"lamp\":{\"LAMP_CONF\":\"/var/lib/lamp/conf\"}}"}}`)}}} + lampRuns := `{"id":"lamp.bundle-tools","type":"archive","source":"` + catalogue.ArtifactStoreScheme + `lamp/tools/blobs/` + dg('b') + + `","digest":"` + dg('b') + `","path":"` + catalogue.BundlePath("lamp", "tools") + `"}` + for _, row := range []struct { + name string + set func(n *Node) delivery.Composed // and what runs of lamp + fps map[string]string + }{ + {"folded at a build of the old form", func(n *Node) delivery.Composed { + n.InOwn = map[string]bool{"lamp": true} + n.RunsOwn = &delivery.Composed{Node: "anchor", Resources: []json.RawMessage{json.RawMessage(lampRuns)}} + return *n.RunsOwn + }, map[string]string{}}, + {"held, its new build not delivered", func(n *Node) delivery.Composed { + n.Held = map[string]bool{"lamp": true} + lr := delivery.Composed{Node: "anchor", Module: "lamp", Resources: []json.RawMessage{json.RawMessage( + strings.Replace(lampRuns, `"type"`, `"module":"lamp","type"`, 1))}} + n.Kept = map[string]catalogue.Manifest{"lamp": Carried(newLamp, "lamp", &lr)} + return lr + }, map[string]string{"lamp": fpFor('1')}}, + } { + tools := aMesh(t) + tools.Catalogue[catalogue.RuntimeModule], tools.Catalogue["artifacts"], tools.Catalogue["lamp"] = runtime, store, lamp + anchor := nodeOf(t, &tools, "anchor") + anchor.Assigned = append(anchor.Assigned, "artifacts", catalogue.RuntimeModule, "lamp") + tools.BusMinted = []string{"anchor." + catalogue.RuntimeModule} + anchor.SealingKey = keyOf(t, "anchor") + anchor.RunsTools = ToolsRunning(append(append([]delivery.Composed{}, runsTools...), row.set(anchor))) + row.fps[overlay.Name], row.fps["artifacts"], row.fps[catalogue.RuntimeModule] = fpFor('a'), fpFor('5'), fpFor('e') + out, _, _, err := CompositionsIn(tools, "anchor", row.fps, Foreseeing) + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + var env map[string]any + for _, c := range out { + for _, raw := range c.Resources { + var r struct { + ID string `json:"id"` + Env map[string]any `json:"env"` + } + if json.Unmarshal(raw, &r) == nil && r.ID == "node-tools.runtime" { + env = r.Env + } + } + } + if env[catalogue.RuntimeToolModules] != served || env[catalogue.RuntimeToolEnv] != `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf"}}` { + t.Errorf("%s: the runtime serves %v with %v, want what runs of lamp, %s", row.name, + env[catalogue.RuntimeToolModules], env[catalogue.RuntimeToolEnv], served) + } + } } // 1943/AC1: the upgrade policies and sources a composition's value may name are the catalogue's own words. diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 632d117..d8dd9f7 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -308,6 +308,8 @@ func input(ctx context.Context, r Reader, mesh Mesh, s Scope, refuse func(module } } if node == s.Node { + // What its runtime serves as it runs: what a module composed as what runs of it contributes there (issue 1126). + n.RunsTools = compose.ToolsRunning(s.Running) for _, module := range s.InOwn { if n.InOwn == nil { n.InOwn = map[string]bool{} diff --git a/go/compose/input.go b/go/compose/input.go index 5b9cba0..dbd2900 100644 --- a/go/compose/input.go +++ b/go/compose/input.go @@ -210,6 +210,12 @@ type Node struct { // (RunsOwn), folded or leaving the fold, is carried there as it runs, whatever its manifest names, so the node's own // never drops it. Held map[string]bool + // RunsTools is, by module, the tools bundles the node's runtime serves of it as it runs (ToolsRunning of what the + // node runs); nil when not known. A module composed as what runs of it (its Stub or Carried: held, or folded at a + // build of the old form) contributes these to the runtime, never its manifest's: a manifest no build resolved has no + // bundles, and a held module's new build's bundles are not delivered (issue 1126, 2026-10-29: shanks's runtime lost + // the folded systemd-resolved, novox's the folded nats). + RunsTools map[string][]catalogue.Bundle } // Place is where a node is, as its placement says (inv.Overlay). From 94cee5d199a049e73f4d533a80f68a5e2c934f2a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 01:36:17 +0100 Subject: [PATCH 02/52] A data flag that differs alone moves no data: the judgement reads the data items first, and says the flag loudly (issue 1126) novox's mongodb and mssql asked a yes for moving their data on a toolchain move: the running side, read back from the kept declaration, carries no data flag, and judge.go returned moves-data on the flag before it compared the data directories and files, so a flag mismatch both raised a false ask and hid a real move. Now moves-data only when a data item (a directory or file within the data places, or a process of a module with data) changes; a flag-only difference is neither, with a WARNING in its words. One row changed and one added in TestJudgeMove. --- go/delivery/judge.go | 17 +++++++++++++---- go/delivery/judge_test.go | 6 +++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/go/delivery/judge.go b/go/delivery/judge.go index f030401..1aef5da 100644 --- a/go/delivery/judge.go +++ b/go/delivery/judge.go @@ -3,6 +3,7 @@ package delivery import ( "bytes" "encoding/json" + "fmt" "path" "sort" "strings" @@ -84,9 +85,6 @@ func judged(from, to Composed) (Judgement, string) { if ids := changedOf(before, after, func(kind string) bool { return kind == string(declaration.TypeContainer) }); len(ids) > 0 { return JudgedRecreates, "it recreates " + strings.Join(ids, ", ") } - if from.Data != to.Data { - return JudgedMovesData, "its data declaration changes" - } places := dataPlaces(before, after) held := func(kind string) bool { return kind == string(declaration.TypeDirectory) || kind == string(declaration.TypeFile) @@ -104,9 +102,20 @@ func judged(from, to Composed) (Judgement, string) { if to.Data || from.Data { moved = append(moved, changedOf(before, after, func(kind string) bool { return kind == string(declaration.TypeProcess) })...) } + // A data flag that differs is no move of data by itself: data moves only when a data item does (issue 1126, 2026-10-29: + // a running composition read back from the kept declaration carries no flag, and mongodb's toolchain move asked a yes for + // moving its data). It is said loudly in the words, and never hides an item that moved. if len(moved) > 0 { sort.Strings(moved) - return JudgedMovesData, "it changes where its data is kept: " + strings.Join(moved, ", ") + words := "it changes where its data is kept: " + strings.Join(moved, ", ") + if from.Data != to.Data { + words += fmt.Sprintf("; its data flag differs too (running %v, composed %v)", from.Data, to.Data) + } + return JudgedMovesData, words + } + if from.Data != to.Data { + return JudgedNeither, fmt.Sprintf("it recreates no container and moves no data; WARNING: its data flag differs "+ + "(running %v, composed %v) while no data item does, so one side's flag is wrong", from.Data, to.Data) } return JudgedNeither, "it recreates no container and moves no data" } diff --git a/go/delivery/judge_test.go b/go/delivery/judge_test.go index 4e61fd5..2ffa7c1 100644 --- a/go/delivery/judge_test.go +++ b/go/delivery/judge_test.go @@ -48,7 +48,8 @@ func TestJudgeMove(t *testing.T) { {"a container gone", mail('a', true, container, spool), mail('b', true, spool), JudgedRecreates, false}, {"its spool moved", mail('a', true, container, spool), mail('b', true, container, spoolMoved), JudgedMovesData, false}, {"a file in its spool", mail('a', true, container, spool, config), mail('b', true, container, spool, config2), JudgedMovesData, false}, - {"its data declaration", mail('a', false, container, spool), mail('b', true, container, spool), JudgedMovesData, false}, + {"its data flag alone", mail('a', false, container, spool), mail('b', true, container, spool), JudgedNeither, false}, + {"its data flag and its spool", mail('a', false, container, spool), mail('b', true, container, spoolMoved), JudgedMovesData, false}, {"a process of a module with data", mail('a', true, spool, process), mail('b', true, spool, process2), JudgedMovesData, false}, {"a process of a module without", mail('a', false, process), mail('b', false, process2), JudgedNeither, false}, {"its tools only", mail('a', true, container, spool, tools), mail('b', true, container, spool, tools2), JudgedNeither, false}, @@ -61,6 +62,9 @@ func TestJudgeMove(t *testing.T) { if got.Judgement != c.want || got.UserListOnly != c.userList || got.Words == "" { t.Errorf("%s: %+v, want %s (user list only %v)", c.name, got, c.want, c.userList) } + if strings.Contains(c.name, "data flag alone") && !strings.Contains(got.Words, "WARNING: its data flag differs") { + t.Errorf("%s: the flag's difference is not said: %q", c.name, got.Words) + } } if got := JudgeMove(nil, Composed{Node: "ace"}); got.Judgement != JudgedNeither { t.Errorf("the node's own composition: %+v", got) From 22c9ddd32b6a3ba842ccacc85a4b8d209c3e77cc Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 01:37:47 +0100 Subject: [PATCH 03/52] A runtime that already dropped a carried module's tools serves them again, read from what runs and its build (issue 1126, review) rev-held2: ToolsRunning took a carried module's bundles from the runtime's MESH_TOOL_MODULES alone, and shanks had already applied the drop of systemd-resolved, so its runs stayed empty and the drop permanent. Now every bundle archive that runs at its module's BundlePath is a running bundle; one the runtime loads nothing of takes its loads and words from the module's build (its built bundles, else its build's artifacts, by catalogue.BundleLoads, the rule a build's resolution keeps, now one function). The runtime process and each archive are matched by their owning module too, not only by id and path. A row added: the runtime already dropped it. --- go/catalogue/build.go | 46 ++++++++++++++--------- go/compose/composed.go | 73 +++++++++++++++++++++++++------------ go/compose/composed_test.go | 31 +++++++++++----- 3 files changed, 100 insertions(+), 50 deletions(-) diff --git a/go/catalogue/build.go b/go/catalogue/build.go index 91e3460..0a2ff73 100644 --- a/go/catalogue/build.go +++ b/go/catalogue/build.go @@ -126,23 +126,7 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { continue } made := by[a.Name] - // What the runtime loads: what the artifact said, else every entrypoint of a module - // that declares tools, else nothing (the field's own rule; see Artifact.Loads). - // - // **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213). - // A process the host runs is the module's service, not its tools: the controller declares - // the verbs it answers as `tools` and serves them itself, and its bundle would otherwise - // have been launched a second time by the node's runtime, as an MCP child it is not. - loads := append([]string(nil), a.Loads...) - if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] { - loads = append([]string(nil), a.Entrypoints...) - // A bundle compiled to a binary has no entrypoints: the binary is what it is, and what - // the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served - // as Go — the runtime execs it — exactly as a TypeScript one is through its launcher. - if bin := BinaryOf(a); bin != "" { - loads = []string{bin} - } - } + loads := BundleLoads(m, a, run) // **Kept, never routed** (ADR 0155): the builder publishes to the store at the address // it reached it by, and a manifest carrying that address names an installation — // registration refused node-tools for exactly this on 2026-10-02. The build record @@ -715,3 +699,31 @@ func undeliveredBundles(m Manifest) []string { } return problems } + +// BundleLoads is what the runtime loads of the bundle artifact a of m (run: the bundles one of m's own resources runs, +// runByAResource): what the artifact said, else every entrypoint of a module that declares tools, else nothing; a +// bundle compiled to a binary is served by its binary. The rule a build's resolution keeps on Bundle.Loads, and what a +// module composed as what runs of it reads again when the runtime already dropped it (issue 1126). +func BundleLoads(m Manifest, a Artifact, run map[string]bool) []string { + // What the runtime loads: what the artifact said, else every entrypoint of a module + // that declares tools, else nothing (the field's own rule; see Artifact.Loads). + // + // **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213). + // A process the host runs is the module's service, not its tools: the controller declares + // the verbs it answers as `tools` and serves them itself, and its bundle would otherwise + // have been launched a second time by the node's runtime, as an MCP child it is not. + loads := append([]string(nil), a.Loads...) + if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] { + loads = append([]string(nil), a.Entrypoints...) + // A bundle compiled to a binary has no entrypoints: the binary is what it is, and what + // the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served + // as Go — the runtime execs it — exactly as a TypeScript one is through its launcher. + if bin := BinaryOf(a); bin != "" { + loads = []string{bin} + } + } + return loads +} + +// RunByAResource is the bundles of m one of its own resources runs (its service, not its tools). +func RunByAResource(m Manifest) map[string]bool { return runByAResource(m) } diff --git a/go/compose/composed.go b/go/compose/composed.go index 66fc29c..fd15ffe 100644 --- a/go/compose/composed.go +++ b/go/compose/composed.go @@ -140,7 +140,7 @@ func compositions(in Input, node string, fps map[string]string, mode Mode, appli kept := make(map[string]catalogue.Manifest, len(n.Kept)) for m, man := range n.Kept { if man.Stubbed && man.RunsAs == "" { - man.Bundles = n.RunsTools[m] + man.Bundles = runningBundles(n.RunsTools[m], reading.Catalogue[m]) } kept[m] = man } @@ -309,13 +309,16 @@ func RunsAnyOf(runs *delivery.Composed, module string) bool { return false } -// ToolsRunning is, by module, the tools bundles the node's runtime serves as it runs (Node.RunsTools): read from the -// runtime's process in running (catalogue.RuntimeToolModules, catalogue.RuntimeToolEnv, the mesh's own words left out) -// and each bundle's archive there; nil when running holds no runtime process. +// ToolsRunning is, by module, the tools bundles that run on the node (Node.RunsTools): each bundle archive a module's +// resource unpacks at its BundlePath, with what the runtime's own process loads of it and the words it hands it +// (catalogue.RuntimeToolModules, catalogue.RuntimeToolEnv, the mesh's own words left out). A bundle the runtime loads +// nothing of has no Loads: the runtime dropped it, and the composition reads its loads again from the module's build +// (issue 1126, review: shanks had already applied the drop of systemd-resolved). Nil when running holds no runtime process. func ToolsRunning(running []delivery.Composed) map[string][]catalogue.Bundle { runtimeID := catalogue.RuntimeModule + "." + catalogue.RuntimeProcessID() type resource struct { ID string `json:"id"` + Module string `json:"module"` Type string `json:"type"` Path string `json:"path"` Source string `json:"source"` @@ -323,18 +326,25 @@ func ToolsRunning(running []delivery.Composed) map[string][]catalogue.Bundle { Env map[string]string `json:"env"` } var env map[string]string - archives := map[string]resource{} + out := map[string][]catalogue.Bundle{} for _, c := range running { for _, raw := range c.Resources { var r resource if json.Unmarshal(raw, &r) != nil { continue } + owner := r.Module + if owner == "" { + owner, _ = declaration.ModuleGuess(r.ID) + } switch { - case r.Type == "process" && r.ID == runtimeID: + case r.Type == "process" && r.ID == runtimeID && owner == catalogue.RuntimeModule: env = r.Env - case r.Type == "archive" && strings.HasPrefix(r.Path, catalogue.BundleRoot+"/"): - archives[r.Path] = r + case r.Type == "archive" && owner != "" && strings.HasPrefix(r.ID, owner+"."+catalogue.BundleID("")): + name := strings.TrimPrefix(r.ID, owner+"."+catalogue.BundleID("")) + if r.Path == catalogue.BundlePath(owner, name) { + out[owner] = append(out[owner], catalogue.Bundle{Name: name, Source: catalogue.Recorded(r.Source), Digest: r.Digest}) + } } } } @@ -343,15 +353,16 @@ func ToolsRunning(running []delivery.Composed) map[string][]catalogue.Bundle { } var words map[string]map[string]string _ = json.Unmarshal([]byte(env[catalogue.RuntimeToolEnv]), &words) - out := map[string][]catalogue.Bundle{} for _, entry := range strings.Split(env[catalogue.RuntimeToolModules], ",") { module, path, ok := strings.Cut(entry, "=") - if !ok { + if !ok || !strings.HasPrefix(path, catalogue.BundleRoot+"/"+module+"/") { continue } name, load, ok := strings.Cut(strings.TrimPrefix(path, catalogue.BundleRoot+"/"+module+"/"), "/") - if !ok || !strings.HasPrefix(path, catalogue.BundleRoot+"/"+module+"/") { - continue + bundles := out[module] + i := slices.IndexFunc(bundles, func(b catalogue.Bundle) bool { return b.Name == name }) + if !ok || i < 0 { + continue // no archive of it runs: nothing to load } var own map[string]string for w, v := range words[module] { @@ -363,18 +374,8 @@ func ToolsRunning(running []delivery.Composed) map[string][]catalogue.Bundle { } own[w] = v } - bundles := out[module] - i := slices.IndexFunc(bundles, func(b catalogue.Bundle) bool { return b.Name == name }) - if i < 0 { - a, delivered := archives[catalogue.BundlePath(module, name)] - if !delivered { - continue // no archive of it runs: nothing to load - } - bundles = append(bundles, catalogue.Bundle{Name: name, Source: catalogue.Recorded(a.Source), Digest: a.Digest, Env: own}) - i = len(bundles) - 1 - } bundles[i].Loads = append(bundles[i].Loads, load) - out[module] = bundles + bundles[i].Env = own } for _, bundles := range out { sort.Slice(bundles, func(i, j int) bool { return bundles[i].Name < bundles[j].Name }) @@ -382,6 +383,32 @@ func ToolsRunning(running []delivery.Composed) map[string][]catalogue.Bundle { return out } +// runningBundles is the bundles of a module composed as what runs of it (Node.RunsTools): each that runs, with what the +// runtime loads of it and its words; one the runtime loads nothing of takes its loads and words from the module's build +// (man: its built bundles, else its build's artifacts, catalogue.BundleLoads), so a drop already applied is undone. +func runningBundles(runs []catalogue.Bundle, man catalogue.Manifest) []catalogue.Bundle { + var out []catalogue.Bundle + for _, b := range runs { + if len(b.Loads) == 0 { + for _, built := range man.Bundles { + if built.Name == b.Name { + b.Loads, b.Env, b.Launchers, b.Binary = built.Loads, built.Env, built.Launchers, built.Binary + } + } + if man.Build != nil { + run := catalogue.RunByAResource(man) + for _, a := range man.Build.Artifacts { + if a.Kind == catalogue.ArtifactBundle && a.Name == b.Name { + b.Loads, b.Env, b.Binary = catalogue.BundleLoads(man, a, run), a.Env, catalogue.BinaryOf(a) + } + } + } + } + out = append(out, b) + } + return out +} + // NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot // compose such a module folded into its own composition: the build of the old form it runs is held by the controller // alone (issue 1126). diff --git a/go/compose/composed_test.go b/go/compose/composed_test.go index eb37442..a96ea85 100644 --- a/go/compose/composed_test.go +++ b/go/compose/composed_test.go @@ -112,35 +112,46 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) { lamp := catalogue.Manifest{Module: "lamp", Version: "1", Tools: []string{"on"}, Resources: []map[string]any{{"id": "state", "type": "directory", "path": "/var/lib/lamp", "mode": "0700"}}, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "tools", Kind: catalogue.ArtifactBundle, - Language: "go", System: "arch", From: "cmd/lamp-tools"}}}} + Language: "go", System: "arch", From: "cmd/lamp-tools", Env: map[string]string{"LAMP_CONF": "/var/lib/lamp/conf"}}}}} newLamp, err := lamp.Resolve([]catalogue.Built{{Name: "tools", Kind: catalogue.ArtifactBundle, Reference: catalogue.ArtifactStoreScheme + "lamp/tools/blobs/" + dg('c'), Digest: dg('c')}}) if err != nil { t.Fatal(err) } served := "lamp=" + catalogue.BundlePath("lamp", "tools") + "/lamp-tools" - runsTools := []delivery.Composed{{Node: "anchor", Module: catalogue.RuntimeModule, Resources: []json.RawMessage{json.RawMessage( - `{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","env":{"MESH_TOOL_MODULES":"` + - served + `","MESH_TOOL_ENV":"{\"lamp\":{\"LAMP_CONF\":\"/var/lib/lamp/conf\"}}"}}`)}}} + runtimeRuns := func(dropped bool) delivery.Composed { + env := `"MESH_TOOL_MODULES":"` + served + `","MESH_TOOL_ENV":"{\"lamp\":{\"LAMP_CONF\":\"/var/lib/lamp/conf\"}}"` + if dropped { + env = `"MESH_TOOL_MODULES":""` // the drop already applied (shanks, 2026-10-29) + } + return delivery.Composed{Node: "anchor", Module: catalogue.RuntimeModule, Resources: []json.RawMessage{json.RawMessage( + `{"id":"node-tools.runtime","module":"node-tools","type":"process","name":"node-tools","env":{` + env + `}}`)}} + } lampRuns := `{"id":"lamp.bundle-tools","type":"archive","source":"` + catalogue.ArtifactStoreScheme + `lamp/tools/blobs/` + dg('b') + `","digest":"` + dg('b') + `","path":"` + catalogue.BundlePath("lamp", "tools") + `"}` for _, row := range []struct { - name string - set func(n *Node) delivery.Composed // and what runs of lamp - fps map[string]string + name string + set func(n *Node) delivery.Composed // and what runs of lamp + fps map[string]string + dropped bool }{ {"folded at a build of the old form", func(n *Node) delivery.Composed { n.InOwn = map[string]bool{"lamp": true} n.RunsOwn = &delivery.Composed{Node: "anchor", Resources: []json.RawMessage{json.RawMessage(lampRuns)}} return *n.RunsOwn - }, map[string]string{}}, + }, map[string]string{}, false}, + {"folded, the runtime already dropped it", func(n *Node) delivery.Composed { + n.InOwn = map[string]bool{"lamp": true} + n.RunsOwn = &delivery.Composed{Node: "anchor", Resources: []json.RawMessage{json.RawMessage(lampRuns)}} + return *n.RunsOwn + }, map[string]string{}, true}, {"held, its new build not delivered", func(n *Node) delivery.Composed { n.Held = map[string]bool{"lamp": true} lr := delivery.Composed{Node: "anchor", Module: "lamp", Resources: []json.RawMessage{json.RawMessage( strings.Replace(lampRuns, `"type"`, `"module":"lamp","type"`, 1))}} n.Kept = map[string]catalogue.Manifest{"lamp": Carried(newLamp, "lamp", &lr)} return lr - }, map[string]string{"lamp": fpFor('1')}}, + }, map[string]string{"lamp": fpFor('1')}, false}, } { tools := aMesh(t) tools.Catalogue[catalogue.RuntimeModule], tools.Catalogue["artifacts"], tools.Catalogue["lamp"] = runtime, store, lamp @@ -148,7 +159,7 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) { anchor.Assigned = append(anchor.Assigned, "artifacts", catalogue.RuntimeModule, "lamp") tools.BusMinted = []string{"anchor." + catalogue.RuntimeModule} anchor.SealingKey = keyOf(t, "anchor") - anchor.RunsTools = ToolsRunning(append(append([]delivery.Composed{}, runsTools...), row.set(anchor))) + anchor.RunsTools = ToolsRunning([]delivery.Composed{runtimeRuns(row.dropped), row.set(anchor)}) row.fps[overlay.Name], row.fps["artifacts"], row.fps[catalogue.RuntimeModule] = fpFor('a'), fpFor('5'), fpFor('e') out, _, _, err := CompositionsIn(tools, "anchor", row.fps, Foreseeing) if err != nil { From 187fb63439620d80d92746a4e03110c6ec571c63 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:35:16 +0100 Subject: [PATCH 04/52] Add the four health seats, mesh-conditions, mesh-watchdogs, mesh-self-check and mesh-healers, so the controller's health has homes its lease does not hold (issue 1952 S1, ADR 0564 clause 1) --- go/catalogue/health_seats.go | 133 +++++++++++++++++++++++++++++++++++ go/catalogue/seats.go | 7 ++ go/catalogue/seats_test.go | 21 +++++- 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 go/catalogue/health_seats.go diff --git a/go/catalogue/health_seats.go b/go/catalogue/health_seats.go new file mode 100644 index 0000000..79ae2c6 --- /dev/null +++ b/go/catalogue/health_seats.go @@ -0,0 +1,133 @@ +package catalogue + +// **The mesh's health, out of the controller** (novox/hq ADR 0564 clause 1, issue 1952): four seats at mesh scope, each +// held once, each holder a process on its own bus credential that writes only its own state. +// +// - mesh-conditions, held by the module mesh-conditions (the condition keeper): the one writer of the open conditions, +// raised from the observers' findings and cleared when they go (clauses 2, 3, 9, 13); +// - mesh-watchdogs, held by the watchman (mesh-delivery-watchman): the signals as keys that expire, its stuck keys and +// presence as findings, and the dead letters from the lease write (clauses 4, 5); +// - mesh-self-check, held by the module mesh-self-check: the probes, every five minutes (clause 6); +// - mesh-healers, held by the module mesh-healers: the healers, acting on the keeper's events (clauses 7, 8). +// +// Their events keep the controller's names and bodies (go/health), so a reader changes only the seat it hears. + +// The four health seats. +const ( + ConditionsSeat = "mesh-conditions" + WatchdogsSeat = "mesh-watchdogs" + SelfCheckSeat = "mesh-self-check" + HealersSeat = "mesh-healers" +) + +// healthDecision is the record that made the four seats. +const healthDecision = "novox/hq ADR 0564" + +// conditionsVerbs are the condition keeper's verbs (ADR 0564 clauses 3, 9, 12). +func conditionsVerbs() []Verb { + return []Verb{ + {Name: "conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then oldest, " + + "raised from the observers' findings (the watchman, the self-check, the healers, the assigner) and cleared when " + + "the finding goes, never by hand. Given a key, that one whole with its evidence; with history, every transition " + + "lately, the controller's before the take-over included.", + Input: schema(map[string]string{ + "key": "a condition's key: that one whole; with history, only its transitions", + "scope": "only this scope: machine, plan, call, build, merge, provider, seat, bus, core, probe, mesh, delivery or module", + "severity": "only urgent, or only warning", + "machine": "only those about this machine", + "history": "\"true\": every raising, change, silence and clearing lately, oldest first", + "days": "with history: how many days back (default 7, at most 90)", + }, nil, "history")}, + {Name: "silence", Description: "Stop one condition's messages for a while, a hand act with why: it stays open and in " + + "status, and the silence is kept as its transition in history.", + Input: schema(map[string]string{ + "key": "the condition's key", + "for": "how long: 30m, 4h, 2d; at most 7d", + "why": "why, required and recorded", + "cause": "the cause in a word (the condition's kind when absent)", + }, []string{"key", "for", "why"})}, + {Name: "compare", Description: "The keeper's open conditions against the controller's: every key open in one and not " + + "the other, or open in both with another severity, each with its reason; the kinds that end with the controller " + + "and the keys held while their module is delivered listed apart and not counted. The take-over waits for it empty.", + Input: schema(map[string]string{}, nil)}, + {Name: "take-over", Description: "Ask the operator's Approve to switch who the router, the grants part and " + + "mesh-watcher hear from the controller to the keeper; refused while compare shows a difference. On the warrant " + + "the keeper writes its take-over key.", + Input: schema(map[string]string{"why": "why now, shown to the operator"}, []string{"why"})}, + {Name: "hand-back", Description: "Switch the readers back to the controller's conditions, until the controller's " + + "lease is handed away; refused after it.", + Input: schema(map[string]string{"why": "why, kept with the switch"}, []string{"why"})}, + {Name: "resync", Description: "Read every observer's findings again and raise and clear what differs from the open " + + "conditions: the recovery of a keeper that missed a change.", + Input: schema(map[string]string{}, nil)}, + } +} + +// watchdogsVerbs are the watchman's verbs (ADR 0564 clauses 4, 5, 12). +func watchdogsVerbs() []Verb { + return []Verb{ + {Name: "signals", Description: "Every watchdog of the signals table the watchman keeps, each a key that expires unless " + + "its signal renews it: its row, what renews it, its bound, the age of its newest signal, and the finding its " + + "expiry wrote.", + Input: schema(map[string]string{"row": "one row, e.g. S1"}, nil)}, + {Name: "stuck", Description: "Every want, build or step the watchman says stopped moving past its bound, with since " + + "when and the words of its finding.", + Input: schema(map[string]string{"machine": "only those on this machine"}, nil)}, + {Name: "presence", Description: "Every machine and tool runner the watchman sees connected, with its last word, and " + + "those it says silent; a machine that said it sleeps is shown as sleeping, not silent.", + Input: schema(map[string]string{"machine": "one machine"}, nil)}, + {Name: "dead-letters", Description: "Every message a consumer on the bus gave up on, kept in DEAD_LETTERS: whose " + + "consumer, the subject, how often it was handed over and when it was given up, newest first. With id: that one " + + "whole. With deliver or drop, a hand act with why, taken only once the controller's lease is handed away; " + + "before, refused naming the controller's verb.", + Input: schema(map[string]string{ + "id": "a dead letter's id, as the list gives it: that one whole, with its message", + "consumer": "a consumer's name, or .: only what that one gave up on", + "limit": "how many to list, newest first (default 50); only when listing", + "deliver": "a dead letter's id: deliver it again to the consumer that gave it up (needs why)", + "drop": "a dead letter's id: drop it for good (needs why)", + "why": "with deliver or drop: why, required and recorded", + "cause": "with deliver or drop: the cause in a word (dead-letter when absent)", + }, nil)}, + {Name: "resync", Description: "Rebuild presence from the bus's connections, and the pulses and findings from the keys " + + "the watchman watches, and take every dead-letter notice waiting: the recovery of a watchman that missed a change.", + Input: schema(map[string]string{}, nil)}, + } +} + +// selfCheckVerbs are the self-check's verbs (ADR 0564 clause 6). +func selfCheckVerbs() []Verb { + return []Verb{ + {Name: "doctor", Description: "The self-check: the last run's verdict at once, each probe passed, failed, deferred " + + "(its facts not on the bus yet, named) or could not run, and how long ago. With run, a run now; with probe, one " + + "probe's verdict whole. Runs every five minutes on its own; each failure is a finding.", + Input: schema(map[string]string{ + "run": "\"true\": run every probe now and answer the verdict", + "probe": "one probe's id (D6): its verdict alone, with every finding", + }, nil, "run")}, + {Name: "probes", Description: "The registry: what each probe asserts, where it reads its facts, and the finding it writes.", + Input: schema(map[string]string{}, nil)}, + } +} + +// healersVerbs are the healers' verbs (ADR 0564 clauses 7, 8). +func healersVerbs() []Verb { + return []Verb{ + {Name: "healers", Description: "The healers: each registered response to one kind of condition, its repair, its " + + "budget and what happens when it is spent; every act taken lately with its outcome, the acts it would have " + + "taken while the controller's lease names a controller, and the mesh-wide brake.", + Input: schema(map[string]string{"days": "how many days of acts back (default 7)"}, nil)}, + {Name: "heal", Description: "Take one healer's act for one open condition now, within its budget and the brake.", + Input: schema(map[string]string{ + "key": "the condition's key", + "why": "why now, kept with the act", + }, []string{"key", "why"})}, + {Name: "drill", Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with " + + "what it tests. A drill is never a repair: no healer is wanted for it however often it is run.", + Input: schema(map[string]string{ + "what": "what was done on purpose, in a line", + "why": "what the drill tests", + "condition": "the key of the condition the drill is meant to raise, if any (optional)", + }, []string{"what", "why"})}, + } +} diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 56637b8..3082ce8 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -280,6 +280,13 @@ var defaultSeats = append([]Seat{ {Name: PlacementSeat, Scope: ScopeMesh, Serves: placementVerbs(), Decision: "novox/hq ADR 0567"}, // The delivery's checker (novox/hq ADR 0547 §10, issue 774): the try is its verb, not the controller's. {Name: CheckerSeat, Scope: ScopeMesh, Serves: checkerVerbs(), Decision: "novox/hq ADR 0547"}, + // The mesh's health, out of the controller (novox/hq ADR 0564 clause 1, health_seats.go): the condition keeper, the + // watchman's watchdogs, the self-check and the healers. + {Name: ConditionsSeat, Scope: ScopeMesh, Decision: healthDecision, + Emits: []string{"condition-raised", "condition-changed", "condition-cleared"}, Serves: conditionsVerbs()}, + {Name: WatchdogsSeat, Scope: ScopeMesh, Decision: healthDecision, Serves: watchdogsVerbs()}, + {Name: SelfCheckSeat, Scope: ScopeMesh, Decision: healthDecision, Emits: []string{"checked"}, Serves: selfCheckVerbs()}, + {Name: HealersSeat, Scope: ScopeMesh, Decision: healthDecision, Emits: []string{"healer-acted"}, Serves: healersVerbs()}, // Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a // delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight. // They keep their names until that migration is done deliberately, apart from the node-* pass. diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index 9113bd7..bd22358 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -64,6 +64,20 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { if strings.Join(checkerVerbs, ",") != "try,try-read,try-end" { t.Errorf("the checker seat serves %v", checkerVerbs) } + // The four health seats (novox/hq ADR 0564 clause 1): at mesh scope, each held once, each naming ADR 0564, with the + // verbs and events the record names; the events keep the controller's names (clause 3). + for _, c := range []struct{ seat, verbs, emits string }{ + {ConditionsSeat, "conditions,silence,compare,take-over,hand-back,resync", "condition-raised,condition-changed,condition-cleared"}, + {WatchdogsSeat, "signals,stuck,presence,dead-letters,resync", ""}, + {SelfCheckSeat, "doctor,probes", "checked"}, + {HealersSeat, "healers,heal,drill", "healer-acted"}, + } { + s, ok := SeatNamed(c.seat) + if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || s.Decision != "novox/hq ADR 0564" || + strings.Join(VerbNames(s.Serves), ",") != c.verbs || strings.Join(s.Emits, ",") != c.emits { + t.Errorf("the health seat %s is %v, %+v", c.seat, ok, s) + } + } // The node seat node-engine is the node's engine's (novox/hq ADR 0563 §1): at node scope, delivering nothing, its six // verbs and three events, held by no assignment, and a module's claim of it refused. if engine, ok := SeatNamed(EngineSeat); !ok || engine.Scope != ScopeNode || engine.Delivers != "" || !engine.HeldByTheEngine { @@ -99,9 +113,10 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203, // ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired // mesh-build-machine row goes, when no registered manifest claims it. - // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1). - if len(Seats()) != 49 { - t.Errorf("the mesh defines %d seats rather than 49; the set is closed, so a change here is "+ + // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1); + // fifty-three with the four health seats (novox/hq ADR 0564 clause 1). + if len(Seats()) != 53 { + t.Errorf("the mesh defines %d seats rather than 53; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames()) } } From 4fff214eeb8e4a92c313ef220197ff165b8dce21 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:35:16 +0100 Subject: [PATCH 05/52] Give the condition keeper the controller's condition bodies, the findings layout and the take-over key, so readers change only the seat they hear (issue 1952 S2-S4, ADR 0564 clauses 2, 3, 7-9) --- go/health/condition.go | 288 +++++++++++++++++++ go/health/condition_test.go | 145 ++++++++++ go/health/findings.go | 341 +++++++++++++++++++++++ go/health/health_test.go | 55 ++-- go/health/testdata/condition-events.json | 222 +++++++++++++++ 5 files changed, 1022 insertions(+), 29 deletions(-) create mode 100644 go/health/condition.go create mode 100644 go/health/condition_test.go create mode 100644 go/health/findings.go create mode 100644 go/health/testdata/condition-events.json diff --git a/go/health/condition.go b/go/health/condition.go new file mode 100644 index 0000000..987cf1c --- /dev/null +++ b/go/health/condition.go @@ -0,0 +1,288 @@ +package health + +// **A condition and its events, as the controller says them today** (novox/hq ADR 0564 clause 3, issue 1952): the body +// of `condition-raised`, `condition-changed` and `condition-cleared`, ported field for field from mesh-controller +// internal/conditions (condition.go, events.go, plain.go at 0fbb5ef3), so the condition keeper says them unchanged under +// the seat mesh-conditions and a reader (the router, the grants part, the healers) changes only the seat it hears. +// +// A condition is a durable fact about something the mesh owns that is wrong: raised when an observer's finding says so, +// kept with since-when, evidence and who can resolve it, said on the bus as it changes, and cleared when the finding +// goes, never by hand (ADR 0231 §14, ADR 0564 clause 2). + +import ( + "regexp" + "sort" + "strings" + "time" +) + +// Severity is how soon the operator is needed: two levels, no more. +type Severity string + +const ( + // Urgent needs the operator now. + Urgent Severity = "urgent" + // Warning needs the operator when they can. + Warning Severity = "warning" +) + +// The scopes a condition's key starts with: what kind of thing is wrong. +const ( + ScopeMachine = "machine" + ScopePlan = "plan" + ScopeCall = "call" + ScopeBuild = "build" + ScopeMerge = "merge" + ScopeProvider = "provider" + ScopeSeat = "seat" + ScopeBus = "bus" + ScopeCore = "core" + ScopeProbe = "probe" + ScopeMesh = "mesh" + // ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239). + ScopeDelivery = "delivery" + // ScopeModule is a module on a machine, by `.` (novox/hq ADR 0240). + ScopeModule = "module" +) + +// Scopes is every scope, in the order a person reads them. +var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider, + ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery, ScopeModule} + +// Who resolves a condition. +const ( + // ResolverSelf clears on observation: the signal returns, the probe passes. + ResolverSelf = "self" + // ResolverOperator needs a person: a healer's budget spent, or a repair that could only destroy. + ResolverOperator = "operator" + // ResolverAgent is work handed to an agent. + ResolverAgent = "agent" +) + +// ResolverHealer is the resolver of a condition a registered healer works on. +func ResolverHealer(name string) string { return "healer:" + name } + +// Subject is what the condition is about: its scope, its id within the scope, and the machine it concerns when there is +// one. +type Subject struct { + Scope string `json:"scope"` + ID string `json:"id"` + Machine string `json:"machine,omitempty"` + // Also are the other machines it concerns: a consumer's, for a provider failing it. + Also []string `json:"also,omitempty"` +} + +// Evidence is one observation, as it was said. +type Evidence struct { + At time.Time `json:"at"` + Said string `json:"said"` +} + +// Tried is one healer's try at a condition, as the condition carries it in `tried`: when, what it did, what came of it, +// and which healer (`healer H3`), never a person. The keeper writes it from the healers' attempts (ADR 0564 clause 7). +type Tried struct { + At time.Time `json:"at"` + What string `json:"what"` + Outcome string `json:"outcome"` + By string `json:"by"` +} + +// Silence is a person saying they know: no messages until it ends. The condition stays open. +type Silence struct { + Until time.Time `json:"until"` + By string `json:"by"` + Why string `json:"why"` + Since time.Time `json:"since"` +} + +// Gap is a stretch in which a fault was cleared, between its clearing and its reopening. +type Gap struct { + Cleared time.Time `json:"cleared"` + Reopened time.Time `json:"reopened"` +} + +// Action is one answer the operator may give from a notification: a label, and the seat verb the operator channel calls +// with these arguments when it is chosen (novox/hq ADR 0253). Level is how much proof its answer needs (ADR 0234 §8, +// ADR 0259). +type Action struct { + Label string `json:"label"` + Verb string `json:"verb"` + Machine string `json:"machine,omitempty"` + Arguments map[string]string `json:"arguments,omitempty"` + Level string `json:"level,omitempty"` +} + +// The assurance levels an action's answer needs (novox/hq ADR 0234 §8). +const ( + LevelAcknowledge = "acknowledge" + LevelApprove = "approve" +) + +// The bounds a condition keeps, as the controller keeps them. +const ( + // KeptEvidence is how many observations a condition keeps, newest first. + KeptEvidence = 10 + // KeptTried is how many tries a condition keeps, newest last. + KeptTried = 10 + // KeptGaps is how many gaps a condition keeps. + KeptGaps = 8 + // MaxSilence is the longest a condition may be silenced at once. + MaxSilence = 7 * 24 * time.Hour + // ReopenWithin is how soon after it cleared a condition raised again is the same one again, its count increased, + // rather than news. + ReopenWithin = 10 * time.Minute +) + +// Condition is one open condition, as the keeper keeps it and its events carry it: the controller's body, field for +// field (ADR 0564 clause 3). The keeper writes Epoch 0 (Limits 10). +type Condition struct { + Key string `json:"key"` + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity Severity `json:"severity"` + Summary string `json:"summary"` + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Resolved string `json:"resolved"` + Needs string `json:"needs"` + Actions []Action `json:"actions"` + Evidence []Evidence `json:"evidence"` + // Source is the observer's row, probe or event that raised it: `S1`, `D3`, `provisioner.failing`. + Source string `json:"source"` + Raised time.Time `json:"raised"` + LastObserved time.Time `json:"last-observed"` + // First and Gaps are the fault's earlier life: a reopening within ReopenWithin is the same fault. + First time.Time `json:"first,omitzero"` + Gaps []Gap `json:"gaps,omitempty"` + Observations int `json:"observations"` + Count int `json:"count"` + Tried []Tried `json:"tried,omitempty"` + Resolver string `json:"resolver"` + // Silenced is null when no silence is in force: said, not left out. + Silenced *Silence `json:"silenced"` + Epoch uint64 `json:"epoch"` + // LooksUnread says why Observations, LastObserved and Evidence are not known; the controller's alone. + LooksUnread string `json:"looks-unread,omitempty"` +} + +// SilencedAt says whether a person's silence is in force at a moment. +func (c Condition) SilencedAt(now time.Time) bool { + return c.Silenced != nil && now.Before(c.Silenced.Until) +} + +// Escalated says a healer tried and its budget is spent: the operator resolves it now. +func (c Condition) Escalated() bool { return c.Resolver == ResolverOperator && len(c.Tried) > 0 } + +// Began is when this fault began as the mesh knows it: its first raising, a reopening within ReopenWithin being the +// same fault again. +func (c Condition) Began() time.Time { + if !c.First.IsZero() && c.First.Before(c.Raised) { + return c.First + } + return c.Raised +} + +// ShowKeeper is the verb that shows more about a condition the keeper says (ADR 0564 Limits 9); a condition carried from +// the controller's bucket keeps the controller's (ShowController) until the lease write. +func ShowKeeper(key string) string { return "mesh-conditions.conditions key=" + key } + +// ShowController is the controller's verb that shows more about a condition it raised. +func ShowController(key string) string { return "mesh-controller.conditions key=" + key } + +// SilenceAction is the action that stops a condition's messages for a week, as the keeper offers it (ADR 0564 Limits 9: +// `mesh-conditions.silence`). An acknowledgement (ADR 0234 §8); its cause marks it as an answer. +func SilenceAction(key string) Action { + return Action{Label: "Silence for a week", Verb: "mesh-conditions.silence", Level: LevelAcknowledge, + Arguments: map[string]string{"key": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}} +} + +// CauseOperatorAnswer is the cause an answer chosen on a notification is recorded with. +const CauseOperatorAnswer = "operator-answer" + +// unsafeKey is anything a key may not hold: the bus takes letters, digits and `-_/=` in a key's tokens, and a `*` or `>` +// would make one a wildcard. +var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) + +// Key composes a condition's key from its parts, `..`, the controller's rule (condition.go Key): each +// token made safe for the bus, a character the bus would refuse becoming `_`, an empty token `_`. The id may hold several +// tokens joined by dots. A finding and the condition it raises have one key. +func Key(scope, id, token string) string { + var parts []string + for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { + p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") + if p == "" { + p = "_" + } + parts = append(parts, p) + } + return strings.Join(parts, ".") +} + +// Order sorts conditions as `conditions` says them: urgent before warning, then oldest first, then by key. +func Order(list []Condition) { + sort.SliceStable(list, func(i, j int) bool { + if list[i].Severity != list[j].Severity { + return list[i].Severity == Urgent + } + if !list[i].Raised.Equal(list[j].Raised) { + return list[i].Raised.Before(list[j].Raised) + } + return list[i].Key < list[j].Key + }) +} + +// The events a condition's life emits, under the seat that keeps it (ADR 0564 clause 3: the controller's names). +const ( + // EventRaised: a condition was raised, new or the same fault again within ReopenWithin (Change says which). + EventRaised = "condition-raised" + // EventChanged: its severity, its resolver, its silence or its words changed; not every observation. + EventChanged = "condition-changed" + // EventCleared: the finding went; the condition is removed and the transition kept in history. + EventCleared = "condition-cleared" +) + +// Events is every event a condition's life emits. +var Events = []string{EventRaised, EventChanged, EventCleared} + +// What changed, as an event's Change and a history entry say it. +const ( + ChangeRaised = "raised" + ChangeReopened = "reopened" + ChangeSeverity = "severity" + ChangeResolver = "resolver" + ChangeSilenced = "silenced" + ChangeUnsilenced = "silence-ended" + ChangeCleared = "cleared" + // ChangeWords is what the operator needs, or the answers offered, said differently (novox/hq ADR 0258). + ChangeWords = "words" +) + +// Event is the body of every condition event and the shape a history entry keeps: the condition itself at the top level, +// with what happened to it beside. +type Event struct { + Condition + // Event is the event's own name, so a body read without its subject still says what it is. + Event string `json:"event"` + At time.Time `json:"at"` + // Change is what happened: raised, reopened, severity, resolver, silenced, silence-ended, words, cleared. + Change string `json:"change"` + // Was is the value before, for a severity or resolver change. + Was string `json:"was,omitempty"` + // Why says why it cleared, or why it was silenced. + Why string `json:"why,omitempty"` + // Cleared is when it cleared, on a clearing. + Cleared *time.Time `json:"cleared,omitempty"` + // Show is the verb that shows more. + Show string `json:"show"` +} + +// EventFor is the event a change is said under. +func EventFor(change string) string { + switch change { + case ChangeRaised, ChangeReopened: + return EventRaised + case ChangeCleared: + return EventCleared + } + return EventChanged +} diff --git a/go/health/condition_test.go b/go/health/condition_test.go new file mode 100644 index 0000000..e62ea04 --- /dev/null +++ b/go/health/condition_test.go @@ -0,0 +1,145 @@ +package health + +import ( + "encoding/json" + "os" + "reflect" + "strings" + "testing" + "time" +) + +// ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the +// events the live controller said unchanged. testdata/condition-events.json holds five transitions of the controller's +// condition history (mesh-controller_condition-history, whose entries are the events' bodies, events.go Event), captured +// on 2026-10-29 with `mesh-controller.conditions history=true days=1`: a raising, a reopening with gaps, a resolver and a +// severity change with tried and was, and a clearing with why. Never written by hand. +func TestTheControllersConditionEventsReadUnchanged(t *testing.T) { + raw, err := os.ReadFile("testdata/condition-events.json") + if err != nil { + t.Fatal(err) + } + var captured []json.RawMessage + if err := json.Unmarshal(raw, &captured); err != nil { + t.Fatal(err) + } + changes := map[string]bool{} + for _, body := range captured { + var e Event + if err := json.Unmarshal(body, &e); err != nil { + t.Fatalf("%s: %v", body, err) + } + changes[e.Change] = true + again, err := json.Marshal(e) + if err != nil { + t.Fatal(err) + } + var want, got map[string]any + _ = json.Unmarshal(body, &want) + _ = json.Unmarshal(again, &got) + if !reflect.DeepEqual(want, got) { + t.Errorf("the controller's %s of %s reads back as\n%s\nnot\n%s", e.Change, e.Key, again, body) + } + if EventFor(e.Change) != e.Event || e.Key != Key(e.Subject.Scope, e.Subject.ID, e.Key[strings.LastIndex(e.Key, ".")+1:]) { + t.Errorf("%s: said as %s for a %s, or its key is not its subject's", e.Key, e.Event, e.Change) + } + } + for _, c := range []string{ChangeRaised, ChangeReopened, ChangeResolver, ChangeSeverity, ChangeCleared} { + if !changes[c] { + t.Errorf("no captured %s", c) + } + } + // The keeper's own body (Limits 9, 10): epoch 0, said and not left out, silenced null, and its show and silence + // naming the new seat. + c := Condition{Key: "machine.g14.silent", Actions: []Action{SilenceAction("machine.g14.silent")}} + b, _ := json.Marshal(Event{Condition: c, Event: EventRaised, Change: ChangeRaised, Show: ShowKeeper(c.Key)}) + for _, says := range []string{`"epoch":0`, `"silenced":null`, `"show":"mesh-conditions.conditions key=machine.g14.silent"`, + `"verb":"mesh-conditions.silence"`} { + if !strings.Contains(string(b), says) { + t.Errorf("the keeper's body %s does not say %s", b, says) + } + } +} + +// ADR 0564 clauses 2, 7, 8, 9 (issue 1952 S3, S4): the layouts between the observers, the healers and the keeper, and the +// take-over key the readers follow. +func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { + at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing", + Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, + Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} + for _, c := range []struct { + name string + edit func(*Finding) + fails string + }{ + {name: "a finding keyed by the controller's key rule"}, + // The key rule is the controller's (condition.go Key): a character the bus refuses becomes `_`, so the finding + // and the condition it raises share one key. + {name: "an unsafe id", edit: func(f *Finding) { + f.Subject.ID, f.Key = "keycloak.ace.mesh mail*", "provider.keycloak.ace.mesh_mail_.failing" + }}, + {name: "a key not its parts'", edit: func(f *Finding) { f.Key = "provider.keycloak.failing" }, fails: "not"}, + {name: "an unknown scope", edit: func(f *Finding) { f.Subject.Scope = "thing" }, fails: "scope"}, + {name: "no severity", edit: func(f *Finding) { f.Severity = "" }, fails: "urgent or a warning"}, + {name: "no observer", edit: func(f *Finding) { f.Observer = "" }, fails: "observer"}, + {name: "eleven observations", edit: func(f *Finding) { f.Evidence = make([]Evidence, KeptEvidence+1) }, fails: "at most"}, + } { + f := good + if c.edit != nil { + c.edit(&f) + } + b, err := EncodeFinding(f) + if c.fails != "" { + if err == nil || !strings.Contains(err.Error(), c.fails) { + t.Errorf("%s: encoded (%v), want refused saying %q", c.name, err, c.fails) + } + continue + } + back, err := DecodeFinding(b) + if err != nil || back.Key != f.Key || back.V != FindingVersion || back.Evidence == nil { + t.Errorf("%s: read back as %+v, %v", c.name, back, err) + } + } + if FindingsBucket(SelfCheckModule) != "mesh-self-check_findings" || Bucket(HealersModule, AttemptsState) != "mesh-healers_attempts" || + Bucket(KeeperModule, TakeoverState) != "mesh-conditions_takeover" { + t.Errorf("the buckets are named %s, %s", FindingsBucket(SelfCheckModule), Bucket(HealersModule, AttemptsState)) + } + + // An attempt reaches the condition's tried as the controller wrote it: by the healer, its outcome and what it said. + if tr := (Attempt{Healer: "H3", Act: "asserted the grants", Outcome: OutcomeEscalated, Said: "budget spent", Begun: at}).Tried(); tr != + (Tried{At: at, What: "asserted the grants", Outcome: "escalated: budget spent", By: "healer H3"}) { + t.Errorf("an attempt is tried as %+v", tr) + } + + // The take-over key (clause 9): absent follows the controller; the keeper's names who chose; anything else refused. + for _, c := range []struct { + raw string + found bool + seat, events string + fails bool + }{ + {found: false, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"}, + {raw: `{"v":1,"follow":"controller","by":"the operator"}`, found: true, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"}, + {raw: `{"v":1,"follow":"keeper","by":"the operator","ask":"ask-1"}`, found: true, seat: "mesh-conditions", events: "mesh-self-check.checked"}, + {raw: `{"v":1,"follow":"keeper"}`, found: true, fails: true}, + {raw: `{"v":1,"follow":"both","by":"x"}`, found: true, fails: true}, + {raw: `{"v":2,"follow":"keeper","by":"x"}`, found: true, fails: true}, + } { + got, err := DecodeTakeover([]byte(c.raw), c.found) + if c.fails != (err != nil) { + t.Errorf("%q: read as %+v, %v", c.raw, got, err) + continue + } + if c.fails { + continue + } + seat, event := got.SelfCheckEvent() + if got.Seat() != c.seat || seat+"."+event != c.events { + t.Errorf("%q follows %s and %s.%s", c.raw, got.Seat(), seat, event) + } + if b, err := EncodeTakeover(got); err != nil || !strings.Contains(string(b), `"v":1`) { + t.Errorf("%q encodes as %s, %v", c.raw, b, err) + } + } +} diff --git a/go/health/findings.go b/go/health/findings.go new file mode 100644 index 0000000..393a52f --- /dev/null +++ b/go/health/findings.go @@ -0,0 +1,341 @@ +package health + +// **What the observers write and the keeper reads** (novox/hq ADR 0564 clauses 2, 6, 7, 8 and 9; issue 1952): the +// layout of a finding, the self-check's `checked`, the healers' `healer-acted`, an act begun in `attempts` and one the +// healers would have taken, and the take-over key the readers follow. The keeper is the one writer of conditions; an +// observer writes findings in its own bucket and never another's (clause 2). + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// The modules holding the four health seats, and the states between them, as `_` buckets. +const ( + // KeeperModule is the condition keeper, holder of mesh-conditions. + KeeperModule = "mesh-conditions" + // SelfCheckModule holds mesh-self-check; HealersModule mesh-healers; WatchmanModule mesh-watchdogs. + SelfCheckModule = "mesh-self-check" + HealersModule = "mesh-healers" + WatchmanModule = "mesh-delivery-watchman" + + // FindingsState is every observer's own state of findings, one key per finding (clause 2). + FindingsState = "findings" + // ConditionsState is the keeper's open conditions, one key each, without their looks (clause 3). + ConditionsState = "conditions" + // TakeoverState is the keeper's take-over, one key TakeoverKey (clause 9). + TakeoverState = "takeover" + // TakeoverKey is the take-over state's one key. + TakeoverKey = "switch" + // AttemptsState is the healers' acts begun, each created before it is made (clause 7); WouldState the acts they + // would have taken while the lease names a controller (clause 8), which the keeper never reads. + AttemptsState = "attempts" + WouldState = "would" +) + +// Bucket is a module's state as the bus names it, `_`. +func Bucket(module, state string) string { return module + "_" + state } + +// FindingsBucket is an observer's bucket of findings. +func FindingsBucket(observer string) string { return Bucket(observer, FindingsState) } + +// FindingVersion is the version of a finding's layout. +const FindingVersion = 1 + +// Finding is one thing an observer sees wrong now, kept as the key Key in its bucket `findings` until it no longer sees +// it (clause 2): written only when what it says changes (ADR 0289 §1, §4), deleted when it goes. A key written raises or +// changes the condition of the same key; a key deleted clears it. The keeper makes the condition from it: its words, +// severity, subject and source unchanged, its evidence kept newest first. +type Finding struct { + V int `json:"v"` + // Key is Key(Subject.Scope, Subject.ID, Token or Kind): the condition's key. + Key string `json:"key"` + Kind string `json:"kind"` + // Token is the key's last part when it is not Kind (`silent` for a machine). + Token string `json:"token,omitempty"` + Subject Subject `json:"subject"` + Severity Severity `json:"severity"` + // Summary is one line in the mesh's words; Headline, Explanation, Resolved and Needs the plain words of ADR 0253 + // and Actions the answers a notification offers. + Summary string `json:"summary"` + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Resolved string `json:"resolved"` + Needs string `json:"needs,omitempty"` + Actions []Action `json:"actions,omitempty"` + // Evidence is what the observer saw, newest first, at most KeptEvidence. Detail goes here, never in Summary. + Evidence []Evidence `json:"evidence"` + // Source is the row, probe or event that saw it: `S1`, `D6`, `apply-failing`. + Source string `json:"source"` + // Resolver is who resolves it, ResolverSelf when empty. + Resolver string `json:"resolver,omitempty"` + // Observer is the module that wrote it, and Run what of its own saw it last (a self-check run's id, empty for a + // watch): the condition's looks are its finding's and its observer's last run (clause 2). + Observer string `json:"observer"` + Run string `json:"run,omitempty"` + At time.Time `json:"at"` +} + +// FindingKey is the key of a finding about scope and id of kind (the token): the condition's key rule. +func FindingKey(scope, id, token string) string { return Key(scope, id, token) } + +func (f Finding) token() string { + if f.Token != "" { + return f.Token + } + return f.Kind +} + +// Check refuses a finding the keeper could not make a condition of, naming why. +func (f Finding) Check() error { + known := false + for _, s := range Scopes { + known = known || s == f.Subject.Scope + } + switch { + case !known: + return fmt.Errorf("health: a finding's scope is one of %s, not %q", strings.Join(Scopes, ", "), f.Subject.Scope) + case strings.TrimSpace(f.Subject.ID) == "": + return fmt.Errorf("health: a %s finding names what it is about", f.Subject.Scope) + case strings.TrimSpace(f.Kind) == "": + return fmt.Errorf("health: the finding %s has no kind", f.Key) + case f.Key != Key(f.Subject.Scope, f.Subject.ID, f.token()): + return fmt.Errorf("health: the finding's key %q is not %q, its scope, id and kind", f.Key, + Key(f.Subject.Scope, f.Subject.ID, f.token())) + case f.Severity != Urgent && f.Severity != Warning: + return fmt.Errorf("health: the finding %s is urgent or a warning, not %q", f.Key, f.Severity) + case strings.TrimSpace(f.Summary) == "": + return fmt.Errorf("health: the finding %s says nothing", f.Key) + case strings.TrimSpace(f.Source) == "": + return fmt.Errorf("health: the finding %s does not say what saw it", f.Key) + case strings.TrimSpace(f.Observer) == "": + return fmt.Errorf("health: the finding %s does not name its observer", f.Key) + case len(f.Evidence) > KeptEvidence: + return fmt.Errorf("health: the finding %s keeps %d observations, at most %d", f.Key, len(f.Evidence), KeptEvidence) + } + return nil +} + +// EncodeFinding is f as JSON at FindingVersion; it refuses what Check refuses. +func EncodeFinding(f Finding) ([]byte, error) { + f.V = FindingVersion + if f.Evidence == nil { + f.Evidence = []Evidence{} + } + if err := f.Check(); err != nil { + return nil, err + } + return json.Marshal(f) +} + +// DecodeFinding reads a finding's value; it refuses another version and what Check refuses. +func DecodeFinding(data []byte) (Finding, error) { + var f Finding + if err := json.Unmarshal(data, &f); err != nil { + return Finding{}, fmt.Errorf("health: a finding is not readable: %w", err) + } + if f.V != FindingVersion { + return Finding{}, fmt.Errorf("health: a finding of version %d, and this reads %d", f.V, FindingVersion) + } + if err := f.Check(); err != nil { + return Finding{}, err + } + return f, nil +} + +// EventChecked is the self-check's event, ending every run (clause 6), with the body of the controller's +// `doctor-heartbeat`, which mesh-watcher hears in its place. +const EventChecked = "checked" + +// The verdicts of a probe in a run. +const ( + VerdictPassed = "passed" + VerdictFailed = "failed" + VerdictFailedToRun = "failed-to-run" + VerdictDeferred = "deferred" +) + +// ProbeVerdict is one probe's outcome in a run (mesh-controller doctor.go probeVerdict). +type ProbeVerdict struct { + ID string `json:"id"` + Verdict string `json:"verdict"` + Found []string `json:"found,omitempty"` + // Unconfirmed are findings one look can be wrong about, seen by this run and not the one before. + Unconfirmed []string `json:"unconfirmed,omitempty"` + Error string `json:"error,omitempty"` + Took string `json:"took,omitempty"` + // FoundCount and More are an overview's, never a checked event's. + FoundCount int `json:"found-count,omitempty"` + More string `json:"more,omitempty"` +} + +// CheckedCounts are a run's verdicts, counted. +type CheckedCounts struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + FailedToRun int `json:"failed-to-run"` + Deferred int `json:"deferred"` +} + +// Checked is one run of the self-check and the body of `checked`, the controller's doctorRun (doctor.go:230) unchanged. +type Checked struct { + Run string `json:"run"` + At time.Time `json:"at"` + Started time.Time `json:"started"` + Took string `json:"took"` + IntervalSeconds int `json:"interval-seconds"` + Counts CheckedCounts `json:"counts"` + Probes []ProbeVerdict `json:"probes"` + // Controller is the machine that ran it (the name kept for mesh-watcher, which reads it), and Why what started it. + Controller string `json:"controller"` + Why string `json:"why"` + Unsaid int `json:"unsaid,omitempty"` +} + +// EventHealerActed is the healers' event, one per act (clause 7). +const EventHealerActed = "healer-acted" + +// The outcomes of an act. +const ( + // OutcomeActing is an act begun and not yet finished: counted against the budget and the brake from its begin. + OutcomeActing = "acting" + // OutcomeActed is an act that did what it does; whether it repaired anything is the condition's clearing to say. + OutcomeActed = "acted" + // OutcomeFailed is an act that could not be done. + OutcomeFailed = "failed" + // OutcomeEscalated is a budget spent, said: the condition handed to the operator. + OutcomeEscalated = "escalated" +) + +// HealerActed is the body of `healer-acted`, the controller's (healers.go:345) unchanged; mesh-healers leaves Epoch out +// (ADR 0564 Limits 10). +type HealerActed struct { + Event string `json:"event"` + At time.Time `json:"at"` + Healer string `json:"healer"` + By string `json:"by"` + Condition string `json:"condition"` + Kind string `json:"kind"` + Act string `json:"act"` + Outcome string `json:"outcome"` + Said string `json:"said"` + Budget string `json:"budget"` + Epoch uint64 `json:"epoch,omitempty"` + Show string `json:"show"` +} + +// Attempt is one act of a healer as kept in the healers' state `attempts` (clause 7): created before it is made, so a +// holder that dies mid-act has spent it and a second instance cannot begin the same act, and finished after. The key is +// the healers' own; the keeper reads every value and writes the condition named its `tried`, resolver and escalation. +type Attempt struct { + Healer string `json:"healer"` + Condition string `json:"condition"` + Kind string `json:"kind"` + // BudgetKey is what the budget counts by: the condition's key unless the healer says another. + BudgetKey string `json:"budget-key"` + Act string `json:"act"` + Outcome string `json:"outcome"` + Said string `json:"said,omitempty"` + Begun time.Time `json:"begun"` + Finished *time.Time `json:"finished,omitempty"` +} + +// Tried is the attempt as the condition carries it in `tried`. +func (a Attempt) Tried() Tried { + outcome := a.Outcome + if a.Said != "" { + outcome += ": " + a.Said + } + return Tried{At: a.Begun, What: a.Act, Outcome: outcome, By: "healer " + a.Healer} +} + +// Would is an act mesh-healers would have taken while the controller's lease names a controller, kept in `would`: it +// counts for no budget, settle or brake and reaches no condition (clause 8). +type Would struct { + Healer string `json:"healer"` + Condition string `json:"condition"` + Kind string `json:"kind"` + Act string `json:"act"` + At time.Time `json:"at"` + // Why says why it was not taken: the lease names a controller, which heals. + Why string `json:"why"` +} + +// Which source the readers of conditions follow (clause 9). +const ( + FollowController = "controller" + FollowKeeper = "keeper" +) + +// TakeoverVersion is the version of the take-over key's value. +const TakeoverVersion = 1 + +// Takeover is the value of the keeper's key `takeover.switch` (clause 9): which source the router, the grants part and +// mesh-watcher hear, the controller's condition events and `doctor-heartbeat`, or the keeper's events and the +// self-check's `checked`. Written by the keeper alone, on the operator's warrant for a take-over and by `hand-back`. An +// absent key is FollowController. +type Takeover struct { + V int `json:"v"` + Follow string `json:"follow"` + At time.Time `json:"at"` + // By is who chose (the operator, as the warrant proves), and Why why; Ask the approve ask it was chosen on. + By string `json:"by"` + Why string `json:"why,omitempty"` + Ask string `json:"ask,omitempty"` +} + +// EncodeTakeover is t as JSON at TakeoverVersion; it refuses a source the readers could not follow. +func EncodeTakeover(t Takeover) ([]byte, error) { + t.V = TakeoverVersion + if err := t.check(); err != nil { + return nil, err + } + return json.Marshal(t) +} + +// DecodeTakeover reads the key as a getter answered it: an absent key follows the controller. +func DecodeTakeover(data []byte, found bool) (Takeover, error) { + if !found || len(data) == 0 { + return Takeover{V: TakeoverVersion, Follow: FollowController}, nil + } + var t Takeover + if err := json.Unmarshal(data, &t); err != nil { + return Takeover{}, fmt.Errorf("health: the take-over key is not readable: %w", err) + } + if t.V != TakeoverVersion { + return Takeover{}, fmt.Errorf("health: the take-over key is of version %d, and this reads %d", t.V, TakeoverVersion) + } + if err := t.check(); err != nil { + return Takeover{}, err + } + return t, nil +} + +func (t Takeover) check() error { + if t.Follow != FollowController && t.Follow != FollowKeeper { + return fmt.Errorf("health: the take-over follows %q, not %s or %s", t.Follow, FollowController, FollowKeeper) + } + if t.Follow == FollowKeeper && strings.TrimSpace(t.By) == "" { + return fmt.Errorf("health: a take-over by the keeper names who chose it") + } + return nil +} + +// Seat is the seat a reader hears condition events on while it follows t: mesh-controller or mesh-conditions. +func (t Takeover) Seat() string { + if t.Follow == FollowKeeper { + return "mesh-conditions" + } + return "mesh-controller" +} + +// SelfCheckEvent is the seat and event a reader of the self-check's run hears while it follows t: the controller's +// `doctor-heartbeat`, or mesh-self-check's `checked`. +func (t Takeover) SelfCheckEvent() (seat, event string) { + if t.Follow == FollowKeeper { + return "mesh-self-check", EventChecked + } + return "mesh-controller", "doctor-heartbeat" +} diff --git a/go/health/health_test.go b/go/health/health_test.go index b6a3531..06310f2 100644 --- a/go/health/health_test.go +++ b/go/health/health_test.go @@ -52,15 +52,6 @@ func TestAWaitIsReadWithWhatItNames(t *testing.T) { } } -func TestAnAnswerWithoutStateIsReadFromHealthy(t *testing.T) { - for raw, want := range map[string]string{`{"healthy":true,"why":"ok"}`: StateHealthy, `{"healthy":false,"why":"down"}`: StateUnhealthy} { - got, err := Read([]byte(raw)) - if err != nil || got.State != want { - t.Fatalf("%s read as %+v, %v", raw, got, err) - } - } -} - func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) { for name, c := range map[string]struct { w []Wait @@ -88,13 +79,6 @@ func TestAWaitThatCannotBeReadIsUnhealthyAndSaysWhy(t *testing.T) { } } -func TestAnUnknownStateIsUnhealthy(t *testing.T) { - got, err := Read([]byte(`{"state":"fine","healthy":true}`)) - if err != nil || got.State != StateUnhealthy || got.Healthy { - t.Fatalf("read as %+v, %v", got, err) - } -} - func TestWaitsBesideAFaultAreSaidAndExcuseNothing(t *testing.T) { got := roundTrip(t, Unhealthy("the share is unreachable", password, Wait{Part: "p", What: "w"})) if got.State != StateUnhealthy || len(got.Waits) != 1 { @@ -111,12 +95,6 @@ func TestWaitingWithNothingIsUnhealthy(t *testing.T) { } } -func TestNotAnAnswerIsAnError(t *testing.T) { - if _, err := Read([]byte(`"yes"`)); err == nil { - t.Fatal("a string read as an answer") - } -} - func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) { for _, words := range []string{ "the source (/mnt/games)", `the source "/mnt/games"`, "the folder ~/games", "the folder ./games", @@ -136,15 +114,34 @@ func TestEveryFormOfPathOrAddressIsRefused(t *testing.T) { } } -func TestHealthyStateWithHealthyFalseIsUnhealthy(t *testing.T) { - got, err := Read([]byte(`{"state":"healthy","healthy":false,"why":"x"}`)) - if err != nil || got.State != StateUnhealthy || !strings.Contains(got.Why, "contradict") { - t.Fatalf("read as %+v, %v", got, err) - } -} - func TestWaitingChecksItsWaits(t *testing.T) { if a := Waiting("it waits", Wait{Part: "p", What: "w"}); a.State != StateUnhealthy || !strings.Contains(a.Why, "neither") { t.Fatalf("Waiting with a malformed wait is %+v", a) } } + +// A raw answer is read by what it says: without state from healthy; an unknown state, or healthy with healthy false, as +// unhealthy; not an answer at all as an error. +func TestARawAnswerIsReadByWhatItSays(t *testing.T) { + for _, c := range []struct { + raw, state, says string + fails bool + }{ + {raw: `{"healthy":true,"why":"ok"}`, state: StateHealthy}, + {raw: `{"healthy":false,"why":"down"}`, state: StateUnhealthy}, + {raw: `{"state":"fine","healthy":true}`, state: StateUnhealthy}, + {raw: `{"state":"healthy","healthy":false,"why":"x"}`, state: StateUnhealthy, says: "contradict"}, + {raw: `"yes"`, fails: true}, + } { + got, err := Read([]byte(c.raw)) + if c.fails { + if err == nil { + t.Errorf("%s read as an answer", c.raw) + } + continue + } + if err != nil || got.State != c.state || (c.state == StateUnhealthy && got.Healthy) || !strings.Contains(got.Why, c.says) { + t.Errorf("%s read as %+v, %v", c.raw, got, err) + } + } +} diff --git a/go/health/testdata/condition-events.json b/go/health/testdata/condition-events.json new file mode 100644 index 0000000..2b25e7e --- /dev/null +++ b/go/health/testdata/condition-events.json @@ -0,0 +1,222 @@ +[ + { + "actions": null, + "at": "2026-10-29T08:41:18.10610281Z", + "change": "raised", + "count": 1, + "epoch": 395230, + "event": "condition-raised", + "evidence": [ + { + "at": "2026-10-29T08:41:18.10610281Z", + "said": "silent for 2m0s" + } + ], + "explanation": "Nothing for you to do. The mesh has not heard from g14 for several minutes. A machine asleep or away is normal.", + "headline": "g14 is not answering", + "key": "machine.g14.silent", + "kind": "silent", + "last-observed": "2026-10-29T08:41:18.10610281Z", + "needs": "", + "observations": 1, + "raised": "2026-10-29T08:41:18.10610281Z", + "resolved": "g14 answers again", + "resolver": "self", + "severity": "warning", + "show": "mesh-controller.conditions key=machine.g14.silent", + "silenced": null, + "source": "S1", + "subject": { + "id": "g14", + "machine": "g14", + "scope": "machine" + }, + "summary": "g14 has not been heard from since 2026-10-29 08:38 UTC (bound 3m0s)" + }, + { + "actions": null, + "at": "2026-10-28T20:59:53.719121257Z", + "change": "reopened", + "count": 2, + "epoch": 387175, + "event": "condition-raised", + "evidence": [ + { + "at": "2026-10-28T20:59:53.719121257Z", + "said": "70 pending ([70 on EVENTS]), last taken 2026-10-28T20:57:25Z" + } + ], + "explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.", + "first": "2026-10-28T20:54:53.510073517Z", + "gaps": [ + { + "cleared": "2026-10-28T20:56:52.499154879Z", + "reopened": "2026-10-28T20:59:53.719121257Z" + } + ], + "headline": "The controller stopped listening", + "key": "core.controller.deaf", + "kind": "controller-deaf", + "last-observed": "2026-10-28T20:59:53.719121257Z", + "needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.", + "observations": 1, + "raised": "2026-10-28T20:59:53.719121257Z", + "resolved": "The controller listens again", + "resolver": "self", + "severity": "urgent", + "show": "mesh-controller.conditions key=core.controller.deaf", + "silenced": null, + "source": "S4", + "subject": { + "id": "controller", + "machine": "novox", + "scope": "core" + }, + "summary": "the controller's event loop has taken nothing for 2m0s while its consumers hold 70 message(s): reports, builds and merges are not being acted on" + }, + { + "actions": null, + "at": "2026-10-29T18:31:48.308721256Z", + "change": "resolver", + "count": 1, + "epoch": 401364, + "event": "condition-changed", + "evidence": [ + { + "at": "2026-10-29T18:31:33.979579505Z", + "said": "no answer for node-files from g14 to the bus's discovery" + } + ], + "explanation": "Nothing for you to do. Whatever serves node-files on g14 does not answer, so its tools do nothing there. A machine asleep is normal.", + "headline": "Node-files on g14 is not answering", + "key": "seat.node-files.g14.silent", + "kind": "holder-silent", + "last-observed": "2026-10-29T18:31:33.979579Z", + "needs": "", + "observations": 1, + "raised": "2026-10-29T18:31:33.979579505Z", + "resolved": "Node-files on g14 answers again", + "resolver": "healer:H3", + "severity": "warning", + "show": "mesh-controller.conditions key=seat.node-files.g14.silent", + "silenced": null, + "source": "D3", + "subject": { + "id": "node-files.g14", + "machine": "g14", + "scope": "seat" + }, + "summary": "node-files's holder on g14 does not answer the bus: its verbs reach nothing there", + "tried": [ + { + "at": "2026-10-29T18:31:48.308721256Z", + "by": "healer H3", + "outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)", + "what": "asserted the bus's streams, consumers and seat workers again" + } + ], + "was": "self", + "why": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)" + }, + { + "actions": null, + "at": "2026-10-28T20:56:51.686413523Z", + "change": "severity", + "count": 1, + "epoch": 387175, + "event": "condition-changed", + "evidence": [ + { + "at": "2026-10-28T20:55:13.087322533Z", + "said": "no answer for mesh-placement from ace to the bus's discovery" + }, + { + "at": "2026-10-28T20:50:11.256396519Z", + "said": "no answer for mesh-placement from ace to the bus's discovery" + } + ], + "explanation": "Needs you: the mesh tried to repair this and could not; read the details to decide what to do. Whatever serves mesh-placement on ace does not answer, so its tools do nothing there. A machine asleep is normal.", + "headline": "Mesh-placement on ace is not answering", + "key": "seat.mesh-placement.ace.silent", + "kind": "holder-silent", + "last-observed": "2026-10-28T20:55:13.087322Z", + "needs": "the mesh tried to repair this and could not; read the details to decide what to do.", + "observations": 2, + "raised": "2026-10-28T20:50:11.256396519Z", + "resolved": "Mesh-placement on ace answers again", + "resolver": "operator", + "severity": "urgent", + "show": "mesh-controller.conditions key=seat.mesh-placement.ace.silent", + "silenced": null, + "source": "D3", + "subject": { + "id": "mesh-placement.ace", + "machine": "ace", + "scope": "seat" + }, + "summary": "mesh-placement's holder on ace does not answer the bus: its verbs reach nothing there", + "tried": [ + { + "at": "2026-10-28T20:50:22.435049179Z", + "by": "healer H3", + "outcome": "acted: every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there (act 1 of 1 within 1h0m0s)", + "what": "asserted the bus's streams, consumers and seat workers again" + }, + { + "at": "2026-10-28T20:56:51.686413523Z", + "by": "healer H3", + "outcome": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)", + "what": "handed to the operator: nothing in the mesh will repair it now" + } + ], + "was": "warning", + "why": "escalated: its budget of 1 act(s) within 1h0m0s is spent and seat.mesh-placement.ace.silent is still open: acted at 20:50 UTC (asserted the bus's streams, consumers and seat workers again \u2014 every object the mesh defines is asserted; the probe that raised it says on its next run whether it is there)" + }, + { + "actions": null, + "at": "2026-10-28T21:18:48.630162912Z", + "change": "cleared", + "cleared": "2026-10-28T21:18:48.630162912Z", + "count": 2, + "epoch": 387463, + "event": "condition-cleared", + "evidence": [ + { + "at": "2026-10-28T21:18:16.589232939Z", + "said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z" + }, + { + "at": "2026-10-28T21:17:46.778840937Z", + "said": "82 pending ([82 on EVENTS]), last taken 2026-10-28T21:15:22Z" + } + ], + "explanation": "Needs you: restart the controller if this stays, from the mesh MCP server; this notification cannot do it. The controller, which coordinates the mesh, has taken no messages for minutes while some wait. Changes and repairs do not happen until it recovers.", + "first": "2026-10-28T21:13:16.626696927Z", + "gaps": [ + { + "cleared": "2026-10-28T21:13:47.735189608Z", + "reopened": "2026-10-28T21:17:46.778840937Z" + } + ], + "headline": "The controller stopped listening", + "key": "core.controller.deaf", + "kind": "controller-deaf", + "last-observed": "2026-10-28T21:18:16.589232Z", + "needs": "restart the controller if this stays, from the mesh MCP server; this notification cannot do it.", + "observations": 2, + "raised": "2026-10-28T21:17:46.778840937Z", + "resolved": "The controller listens again", + "resolver": "self", + "severity": "urgent", + "show": "mesh-controller.conditions key=core.controller.deaf", + "silenced": null, + "source": "S4", + "subject": { + "id": "controller", + "machine": "novox", + "scope": "core" + }, + "summary": "the controller's event loop has taken nothing for 3m0s while its consumers hold 82 message(s): reports, builds and merges are not being acted on", + "why": "S4 no longer observes it" + } +] From 575a05e49e59f96064399df65be34e2e84556d06 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:36:11 +0100 Subject: [PATCH 06/52] Let a state create a key with its own time to live, so a pulse, a settle or a hold bound expires on the bus with no timer (issue 1952 S6, ADR 0564 clauses 4, 7, 13) --- go/process/state.go | 45 ++++++++++++++++++++++++++++ go/process/state_integration_test.go | 18 ++++++++++- go/process/state_test.go | 30 ++++++++++++++++++- 3 files changed, 91 insertions(+), 2 deletions(-) diff --git a/go/process/state.go b/go/process/state.go index 777a9fd..90523b4 100644 --- a/go/process/state.go +++ b/go/process/state.go @@ -113,6 +113,51 @@ func (s State) Create(key string, value any) (uint64, error) { return rev, nil } +// MinTTL is the shortest time to live a key may be given: the bus counts a key's age in whole seconds. +const MinTTL = time.Second + +// CreateTTL writes a key only when it has none, with its own time to live (novox/hq ADR 0564 clauses 4 and 7, issue +// 1952 S6): the bus removes it when ttl passes and its watchers hear the removal, so an expiry is an event and no timer +// runs. A key that has a value is refused as stdio.ErrChanged: an act begun once cannot be begun again. The bucket must +// take keys with their own time to live (a state declared `key-ttl`). +func (s State) CreateTTL(key string, value any, ttl time.Duration) (uint64, error) { + if ttl < MinTTL { + return 0, fmt.Errorf("%s's state %s, key %s: a time to live of %s is shorter than the bus keeps (%s)", s.Module, s.Name, key, ttl, MinTTL) + } + ctx, cancel := context.WithTimeout(context.Background(), stateTimeout) + defer cancel() + kv, err := s.store(ctx) + if err != nil { + return 0, err + } + data, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := kv.Create(ctx, key, data, jetstream.KeyTTL(ttl)) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + if err != nil { + return 0, fmt.Errorf("%s's state %s, key %s: %w", s.Module, s.Name, key, err) + } + return rev, nil +} + +// RenewTTL writes a key whole with a fresh time to live, whatever it held: a pulse its signal renews (ADR 0564 clause +// 4). A time to live is set only when a key is created, so a live key is deleted first; its watchers hear the delete and +// then the new value. +func (s State) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) { + rev, err := s.CreateTTL(key, value, ttl) + if !errors.Is(err, stdio.ErrChanged) { + return rev, err + } + if err := s.Delete(key); err != nil { + return 0, err + } + return s.CreateTTL(key, value, ttl) +} + // Update writes a key only when it is still at revision, and returns the new one; a key changed meanwhile // is refused as stdio.ErrChanged. func (s State) Update(key string, value any, revision uint64) (uint64, error) { diff --git a/go/process/state_integration_test.go b/go/process/state_integration_test.go index bd34388..5c5f07f 100644 --- a/go/process/state_integration_test.go +++ b/go/process/state_integration_test.go @@ -44,7 +44,7 @@ func TestStateCreateGetUpdateOnAGrantedBucket(t *testing.T) { if _, err := awaited.Put(ctxFor(t), "hearing", []byte(`{"heard":true}`)); !errors.Is(err, ErrNotYet) { t.Fatalf("a second write before the bucket exists: %v, want ErrNotYet", err) } - if _, err := adminJetStream.CreateKeyValue(ctxFor(t), jetstream.KeyValueConfig{Bucket: "mesh-mail_sends"}); err != nil { + if _, err := adminJetStream.CreateKeyValue(ctxFor(t), jetstream.KeyValueConfig{Bucket: "mesh-mail_sends", LimitMarkerTTL: time.Second}); err != nil { t.Fatalf("the admin could not make the bucket: %v", err) } select { @@ -94,6 +94,22 @@ func TestStateCreateGetUpdateOnAGrantedBucket(t *testing.T) { if _, err := s.Create("key", map[string]any{"v": 4}); !errors.Is(err, stdio.ErrChanged) { t.Fatalf("Create of an existing key: %v, want stdio.ErrChanged", err) } + + // 1952 S6 (ADR 0564 clause 4): a key created with its own time to live is gone once it passes, on the bus itself. + if _, err := s.CreateTTL("pulse", map[string]any{"v": 5}, time.Second); err != nil { + t.Fatalf("CreateTTL on a bucket that takes keys with a time to live: %v", err) + } + deadline := time.Now().Add(5 * time.Second) + for { + e, err := s.Get("pulse") + if err == nil && e == nil { + break + } + if time.Now().After(deadline) { + t.Fatalf("the key created with a second to live still reads %+v, %v after 5s", e, err) + } + time.Sleep(100 * time.Millisecond) + } } // 1188/AC4: given an account not granted a bucket, when it writes there, then the bus's refusal comes diff --git a/go/process/state_test.go b/go/process/state_test.go index 02b084a..7cc0a93 100644 --- a/go/process/state_test.go +++ b/go/process/state_test.go @@ -38,6 +38,8 @@ type fakeKV struct { // initial is what a Watch hands first, before its nil marker; watcher is the last one made. initial []jetstream.KeyValueEntry watcher *fakeWatcher + // withTTL says the last create of a key carried an option: its time to live. + withTTL map[string]bool } func newFakeKV() *fakeKV { return &fakeKV{values: map[string][]byte{}, revs: map[string]uint64{}} } @@ -52,9 +54,13 @@ func (f *fakeKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, er return fakeEntry{key: key, value: v, rev: f.revs[key]}, nil } -func (f *fakeKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { +func (f *fakeKV) Create(_ context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) { f.mu.Lock() defer f.mu.Unlock() + if f.withTTL == nil { + f.withTTL = map[string]bool{} + } + f.withTTL[key] = len(opts) > 0 if _, ok := f.values[key]; ok { return 0, jetstream.ErrKeyExists } @@ -160,6 +166,28 @@ func TestStateSatisfiesAskerStore(t *testing.T) { if rev2 == rev { t.Fatal("Update did not advance the revision") } + + // 1952 S6 (ADR 0564 clauses 4, 7): a key created with its own time to live is created once, a second create + // refused as an act begun already; a renewal writes it whole with a fresh time to live; a time to live the bus + // cannot keep is refused. + kv := newFakeKV() + s = stateOn(kv) + if _, err := s.CreateTTL("settle.H3", map[string]any{"a": 1}, time.Minute); err != nil || !kv.withTTL["settle.H3"] { + t.Fatalf("CreateTTL: %v, with a time to live %v", err, kv.withTTL["settle.H3"]) + } + if _, err := s.CreateTTL("settle.H3", map[string]any{"a": 2}, time.Minute); !errors.Is(err, stdio.ErrChanged) { + t.Fatalf("CreateTTL of a live key: %v, want stdio.ErrChanged", err) + } + kv.withTTL["settle.H3"] = false + if _, err := s.RenewTTL("settle.H3", map[string]any{"a": 3}, time.Minute); err != nil || !kv.withTTL["settle.H3"] { + t.Fatalf("RenewTTL: %v, with a time to live %v", err, kv.withTTL["settle.H3"]) + } + if e, _ := s.Get("settle.H3"); e == nil || string(e.Value) != `{"a":3}` { + t.Fatalf("after RenewTTL the key reads %+v", e) + } + if _, err := s.CreateTTL("pulse.S1", 1, 10*time.Millisecond); err == nil || !strings.Contains(err.Error(), "shorter") { + t.Fatalf("a time to live under a second: %v", err) + } } // 1840/S1: given a state, when a key is put twice, then each Put returns a new revision and the key reads From 544720b00ab73a3832df4225e0462293971849fa Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:37:59 +0100 Subject: [PATCH 07/52] Read whether a module is being delivered on a node in one place, so the view and the condition keeper judge a delivery alike (issue 1952 S7, ADR 0564 clause 13) --- go/delivery/motion.go | 75 ++++++++++++++++++++++++++++++++++++++ go/delivery/motion_test.go | 58 +++++++++++++++++++++++++++++ go/delivery/stuck_test.go | 32 +++++++--------- 3 files changed, 147 insertions(+), 18 deletions(-) create mode 100644 go/delivery/motion.go create mode 100644 go/delivery/motion_test.go diff --git a/go/delivery/motion.go b/go/delivery/motion.go new file mode 100644 index 0000000..94c94c5 --- /dev/null +++ b/go/delivery/motion.go @@ -0,0 +1,75 @@ +package delivery + +import "time" + +// **Whether a module is being delivered on a node, read from its want and its have** (novox/hq ADR 0564 clause 13 and +// Limits 12, ADR 0540 §7; issue 1952 S7): one derivation, used by the delivery view, which says a module `moving`, +// `settled` or `stuck` (the messenger's hold hears those), and by the condition keeper, which holds a module's own +// findings while it is delivered there and raises what still stands when the delivery ends. Before, it lived in the view +// alone (mesh-delivery-view internal/view motionOf), and a second reader would have judged a delivery its own way. + +// The motion of a module on a node, as the view says it. +const ( + MotionMoving = "moving" + MotionSettled = "settled" + MotionStuck = "stuck" +) + +// The outcomes of a motion that settled or stuck. +const ( + // OutcomeRunning is a settled module: the have reached the want, running and healthy. + OutcomeRunning = "running" + // OutcomeFailed is a have at the wanted fingerprint whose apply or health check failed. + OutcomeFailed = "failed" + // OutcomeStopped is a module held, or a want the watchman says stopped moving past its bound. + OutcomeStopped = "stopped" +) + +// DefaultDeliveryBound is how long a module's delivery holds its own findings when its manifest names no bound (ADR 0564 +// clause 13 (d)): past it, what is held is raised, saying the delivery did not end. +const DefaultDeliveryBound = 15 * time.Minute + +// Motion is where one module's want stands on one node. +type Motion struct { + // State is MotionMoving, MotionSettled or MotionStuck; empty when there is no want to move to (none, removed, or + // naming no fingerprint). + State string + // Outcome and Words are a settled or stuck motion's: running; failed with the have's words; stopped with the have's + // words (held) or the watchman's (stopped moving). + Outcome string + Words string + // Delivering says the module is being delivered there (clause 13 (a)): its want names a fingerprint other than its + // have's, or its have's phase is applying. + Delivering bool + // Failed says the delivery ended failed (Limits 12): the node refused or failed its apply (the have's phase failed, + // at whichever fingerprint it kept), or the have at the wanted fingerprint is unhealthy. What a hold kept is raised + // at once. Apart from State, which says failed only at the wanted fingerprint, as the view always has. + Failed bool +} + +// Ended says the delivery ended, failed or settled healthy at the want: what a hold kept is judged now. +func (m Motion) Ended() bool { return m.Failed || (m.State == MotionSettled && !m.Delivering) } + +// MotionOf is the motion of a module on a node from its want (wanted: one was read), its have (has: one was read) and +// the watchman's words when it says the want stopped moving (stuck: it says so and the want has not moved since). +func MotionOf(w Want, wanted bool, h Have, has bool, stuckWords string, stuck bool) Motion { + if !wanted || w.Removed || w.FP == "" { + return Motion{} + } + m := Motion{Delivering: !has || h.FP != w.FP || h.Phase == PhaseApplying, + Failed: has && (h.Phase == PhaseFailed || (h.FP == w.FP && h.Health == HealthFailed))} + applied := has && h.FP == w.FP && (h.Composed == "" || h.Composed == w.Composed) + switch { + case applied && h.Phase == PhaseRunning && h.Health == HealthOK: + m.State, m.Outcome = MotionSettled, OutcomeRunning + case has && h.Phase == PhaseHeld: + m.State, m.Outcome, m.Words = MotionStuck, OutcomeStopped, h.Detail + case has && h.FP == w.FP && (h.Health == HealthFailed || h.Phase == PhaseFailed): + m.State, m.Outcome, m.Words = MotionStuck, OutcomeFailed, h.Detail + case stuck: + m.State, m.Outcome, m.Words = MotionStuck, OutcomeStopped, stuckWords + default: + m.State = MotionMoving + } + return m +} diff --git a/go/delivery/motion_test.go b/go/delivery/motion_test.go new file mode 100644 index 0000000..42b34ba --- /dev/null +++ b/go/delivery/motion_test.go @@ -0,0 +1,58 @@ +package delivery + +import "testing" + +// 1952 S7 (ADR 0564 clause 13 (a), (c) and Limits 12; ADR 0540 §7): one derivation of whether a module is being +// delivered on a node, read by the view (moving, settled, stuck, as its motionOf said them) and by the condition keeper +// (delivering, failed, ended). Wants and haves as the assigner and the node-engine write them (assigner.go Want, +// delivery.go Have; agent.go writes a refusal as phase failed at the prior fingerprint). +func TestMotionOf(t *testing.T) { + want := Want{V: Version, FP: "fp-new", Composed: "sha256:new"} + at := func(fp, composed string, phase Phase, health Health, detail string) Have { + return Have{V: Version, FP: fp, Composed: composed, Phase: phase, Health: health, Detail: detail} + } + for _, c := range []struct { + name string + w Want + wanted bool + h Have + has bool + stuck string + state string + outcome string + delivering bool + failed bool + ended bool + }{ + {name: "no want", h: at("fp-old", "", PhaseRunning, HealthOK, ""), has: true}, + {name: "a removed want", w: Want{V: Version, Removed: true}, wanted: true}, + {name: "a want no node has yet", w: want, wanted: true, state: MotionMoving, delivering: true}, + {name: "the old fingerprint still runs", w: want, wanted: true, h: at("fp-old", "sha256:old", PhaseRunning, HealthOK, ""), has: true, + state: MotionMoving, delivering: true}, + {name: "applying the wanted fingerprint", w: want, wanted: true, h: at("fp-new", "sha256:new", PhaseApplying, HealthUnknown, ""), has: true, + state: MotionMoving, delivering: true}, + {name: "checking at the want: no longer delivering, not yet settled", w: want, wanted: true, + h: at("fp-new", "sha256:new", PhaseChecking, HealthUnknown, ""), has: true, state: MotionMoving}, + {name: "running healthy at the want", w: want, wanted: true, h: at("fp-new", "sha256:new", PhaseRunning, HealthOK, ""), has: true, + state: MotionSettled, outcome: OutcomeRunning, ended: true}, + {name: "running healthy at another composition", w: want, wanted: true, h: at("fp-new", "sha256:other", PhaseRunning, HealthOK, ""), has: true, + state: MotionMoving}, + {name: "unhealthy at the want", w: want, wanted: true, h: at("fp-new", "sha256:new", PhaseRunning, HealthFailed, "exit 1"), has: true, + state: MotionStuck, outcome: OutcomeFailed, failed: true, ended: true}, + {name: "its apply failed at the want", w: want, wanted: true, h: at("fp-new", "sha256:new", PhaseFailed, HealthUnknown, "no image"), has: true, + state: MotionStuck, outcome: OutcomeFailed, failed: true, ended: true}, + {name: "refused, the prior fingerprint kept", w: want, wanted: true, + h: at("fp-old", "sha256:old", PhaseFailed, HealthOK, "not composed: no grant"), has: true, + state: MotionMoving, delivering: true, failed: true, ended: true}, + {name: "held on the node", w: want, wanted: true, h: at("fp-old", "sha256:old", PhaseHeld, HealthOK, "held by a person"), has: true, + state: MotionStuck, outcome: OutcomeStopped, delivering: true}, + {name: "the watchman says it stopped moving", w: want, wanted: true, h: at("fp-old", "sha256:old", PhaseRunning, HealthOK, ""), has: true, + stuck: "ace to apply fp-new", state: MotionStuck, outcome: OutcomeStopped, delivering: true}, + } { + m := MotionOf(c.w, c.wanted, c.h, c.has, c.stuck, c.stuck != "") + if m.State != c.state || m.Outcome != c.outcome || m.Delivering != c.delivering || m.Failed != c.failed || m.Ended() != c.ended { + t.Errorf("%s: %+v (ended %v), want state %q outcome %q delivering %v failed %v ended %v", c.name, m, m.Ended(), + c.state, c.outcome, c.delivering, c.failed, c.ended) + } + } +} diff --git a/go/delivery/stuck_test.go b/go/delivery/stuck_test.go index fcca151..d204f19 100644 --- a/go/delivery/stuck_test.go +++ b/go/delivery/stuck_test.go @@ -8,8 +8,9 @@ import ( "time" ) +// 1153/AC1: the stuck key is the stopped key's bucket and key, split at the first dot, refused as a pulse key is. // 1153/AC4: the watchman's module and its three buckets, each that module's state. -func TestWatchmanBuckets(t *testing.T) { +func TestStuckKey(t *testing.T) { if WatchmanModule != "mesh-delivery-watchman" { t.Fatalf("WatchmanModule = %q", WatchmanModule) } @@ -18,10 +19,6 @@ func TestWatchmanBuckets(t *testing.T) { t.Errorf("the %s bucket is %q, not %s_%s", name, bucket, WatchmanModule, name) } } -} - -// 1153/AC1: the stuck key is the stopped key's bucket and key, split at the first dot, refused as a pulse key is. -func TestStuckKey(t *testing.T) { key, err := StuckKey(PresenceBucket, "ace") if err != nil || key != "mesh-delivery-watchman_presence.ace" { t.Fatalf("StuckKey = %q, %v", key, err) @@ -47,7 +44,19 @@ func TestStuckKey(t *testing.T) { } // 1153/AC2: a stuck value round-trips with "v":1, and an unknown or missing version is refused loudly. +// 1153/AC3: equal stuck values are the same; one moved is not. func TestStuckValue(t *testing.T) { + { + a := Stuck{V: Version, Bucket: "b", Key: "k", Waited: "w", Since: time.Unix(100, 0).UTC()} + b := a + if !SameStuck(a, b) { + t.Fatal("equal values are not the same") + } + b.Moved = true + if SameStuck(a, b) { + t.Fatal("a moved value is the same as a stuck one") + } + } s := Stuck{Bucket: PresenceBucket, Key: "ace", Waited: "ace to come back online", Since: time.Date(2026, 10, 18, 20, 0, 0, 0, time.UTC)} raw, err := EncodeStuck(s) @@ -73,16 +82,3 @@ func TestStuckValue(t *testing.T) { } } } - -// 1153/AC3: equal stuck values are the same; one moved is not. -func TestSameStuck(t *testing.T) { - a := Stuck{V: Version, Bucket: "b", Key: "k", Waited: "w", Since: time.Unix(100, 0).UTC()} - b := a - if !SameStuck(a, b) { - t.Fatal("equal values are not the same") - } - b.Moved = true - if SameStuck(a, b) { - t.Fatal("a moved value is the same as a stuck one") - } -} From 83d928fe508060ead6f73b0af15ccf4f0d602ed7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:37:59 +0100 Subject: [PATCH 08/52] Refuse mesh-watcher and mesh-self-check on one node, so the watcher of the self-check never falls silent with it (issue 1952 S8, ADR 0564 clause 6, Limits 5) --- go/delivery/placement.go | 19 ++++++++++++++++++- go/delivery/placement_test.go | 5 ++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/go/delivery/placement.go b/go/delivery/placement.go index 60f6e32..d71a3f3 100644 --- a/go/delivery/placement.go +++ b/go/delivery/placement.go @@ -364,12 +364,20 @@ func (p Placement) Held() []catalogue.Held { return out } +// KeptApart are the modules never placed on one node, each row naming the record that asks it (novox/hq ADR 0564 clause 6 +// and Limits 5; issue 1952 S8): no manifest word says it, so the check below holds the table. mesh-watcher hears the +// self-check's `checked` from a second machine (ADR 0227 §6), which on the self-check's own node would fall silent with +// it. +var KeptApart = []struct{ A, B, Record string }{ + {A: "mesh-watcher", B: "mesh-self-check", Record: "ADR 0227 §6, ADR 0564 clause 6"}, +} + // CheckPlacements is the holder's check of a proposal laid over every placement (ADR 0567 §2), over the placements by // node, the modules the indexer holds and the nodes with a record. It returns every refusal, in node order, never only the first: a // placement naming a node not enrolled or a module the catalogue does not hold; two modules of one node on the same host // port; a pin, binding or holding naming a provider that is not placed where it says; a seat of scope mesh held twice, unless the seat is replicated (catalogue.Seat's Replicated, ADR 0223), which // is held once per node; -// two nodes with one address on the private network; more than one hub. +// two nodes with one address on the private network; more than one hub; two modules KeptApart on one node. // A host port a manifest declares without a number and a requirement left unbound are the composition's to find (its // ForeseenPorts and Unbound), which the catalogue's composition check reads. func CheckPlacements(placements map[string]Placement, held func(module string) bool, enrolled func(node string) bool) []string { @@ -411,6 +419,15 @@ func CheckPlacements(placements map[string]Placement, held func(module string) b if !enrolled(node) { refusals = append(refusals, fmt.Sprintf("%s is a node with no record", node)) } + placedHere := map[string]bool{} + for _, m := range p.Modules { + placedHere[m.Module] = true + } + for _, apart := range KeptApart { + if placedHere[apart.A] && placedHere[apart.B] { + refusals = append(refusals, fmt.Sprintf("%s: %s and %s are never placed on one node (%s)", node, apart.A, apart.B, apart.Record)) + } + } hostPorts := map[int]string{} for _, m := range p.Modules { if !held(m.Module) && !ProvidedByMesh(m.Module) { // the mesh provides the private network itself diff --git a/go/delivery/placement_test.go b/go/delivery/placement_test.go index e861328..6ea2dde 100644 --- a/go/delivery/placement_test.go +++ b/go/delivery/placement_test.go @@ -179,6 +179,8 @@ func TestPlacement(t *testing.T) { clash.Modules = append([]PlacedModule{{Module: "ghost", Ports: map[string]int{"80": 18080}}}, g14.Modules...) clash.Modules[2] = PlacedModule{Module: "app", Ports: map[string]int{"8080": 18080}, Bindings: map[string]Provider{"postgres": {Node: "ace", Module: "postgres"}}} clash.Pins = map[string]Provider{"shell": {Node: "g14", Module: "bash"}} + // ADR 0564 clause 6, Limits 5 (issue 1952 S8): mesh-watcher and mesh-self-check are never placed on one node. + clash.Modules = append(clash.Modules, PlacedModule{Module: "mesh-watcher"}, PlacedModule{Module: "mesh-self-check"}) twice := novox twice.Node = "stranger" twice.Decided.Address = "10.77.0.2" @@ -186,7 +188,8 @@ func TestPlacement(t *testing.T) { for _, want := range []string{"holds no module ghost", "ghost and app are both published on host port 18080", "bound to ace/postgres", "pin of shell names g14/bash", "stranger is a node with no record", "mesh-store, held once for the mesh, is held on novox and stranger", - "g14 and stranger both have the address 10.77.0.2", "novox, stranger are each placed as it"} { + "g14 and stranger both have the address 10.77.0.2", "novox, stranger are each placed as it", + "g14: mesh-watcher and mesh-self-check are never placed on one node (ADR 0227 §6, ADR 0564 clause 6)"} { found := false for _, line := range r { found = found || strings.Contains(line, want) From 5a47192b574209fbec7713cdc507ca0d14b86af9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:40:49 +0100 Subject: [PATCH 09/52] Grant the health holders what ADR 0564 gives them and nothing more: named reads of the lease and the controller's conditions and hand acts, the watchman's own dead-letter consumer, and the bus tools' removal account (issue 1952 S5) --- go/buscompose/buscompose_test.go | 96 ++++++++++++++++++++++++++++++++ go/buscompose/health_grants.go | 78 ++++++++++++++++++++++++++ go/buscompose/nats.go | 15 +++++ go/buscompose/userlist_switch.go | 8 ++- go/buscompose/users.go | 2 +- go/buscompose/writers.go | 6 +- 6 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 go/buscompose/health_grants.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index dea1de5..4d23e3f 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -33,8 +33,16 @@ var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATE // grants part's list is the one the bus loads; a hand-back loads the controller's, which lacks these reads, and the nodes // keep applying without composing their own until the next take-over. Each principal's hearing of its buckets' creation // (issue 1790) is the other row beyond, matched by createdAdvisory. +// +// And the watchman's dead letters (novox/hq ADR 0564 clause 5, issue 1952 S5): its own consumer on DEAD_LETTER_NOTICES and +// a read of DEAD_LETTERS, the controller's job until the lease write. func beyondTheControllers(user string) map[string]bool { out := map[string]bool{} + if strings.HasSuffix(user, "."+WatchmanModule) { + for _, s := range WatchmanDeadLetterGrants() { + out[s] = true + } + } node, ok := strings.CutPrefix(user, "node.") if !ok { return out @@ -548,6 +556,8 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { t.Errorf("the grants part creates consumers as %v", u.CreatesConsumers) } } + // 1952 S5 (novox/hq ADR 0564 clauses 2, 5, 8; Limits 2, 4, 8): the health holders' credentials. + healthChecks(t) q := ConsumerConfig(QueueConsumer(Principal{Kind: KindProcess, Node: "a", Module: "m", Process: "p"}, QueueOwner{Process: "p", Stream: "M_WORK", MaxDeliver: 3})) if q.Durable != "a_m_p" || q.FilterSubject != "" || q.MaxDeliver != 3 || q.DeliverPolicy != jetstream.DeliverAllPolicy { @@ -555,3 +565,89 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) { } }) } + +// healthChecks are the health holders' credential cases (ADR 0564 clauses 2, 5, 8; Limits 2, 4, 8; issue 1952 S5). +func healthChecks(t *testing.T) { + t.Helper() + announced := map[string]announce.States{ControllerSeat: ControllerAnnouncement("novox"), + "mesh-healers": {Module: "mesh-healers", States: []announce.State{{Name: "attempts", Readable: true}, {Name: "would", Readable: true}}}} + for _, c := range []struct { + reader, read string + may bool + }{ + {"mesh-conditions", ControllerLease, true}, {"mesh-healers", ControllerLease, true}, {WatchmanModule, ControllerLease, true}, + {"mesh-self-check", ControllerLease, false}, {"messenger", ControllerLease, false}, + {"mesh-conditions", ControllerConditions, true}, {"mesh-healers", ControllerConditions, false}, {"messenger", ControllerConditions, false}, + {"mesh-healers", ControllerHandActs, true}, {"mesh-conditions", ControllerHandActs, false}, + {"mesh-conditions", "mesh-healers.attempts", true}, {"mesh-conditions", HealersWould, false}, {"mesh-delivery-view", HealersWould, false}, + } { + granted, _ := ReadsAnnouncedBy(c.reader, []string{c.read}, announced) + if (len(granted) == 1) != c.may { + t.Errorf("%s reads %s: %v, want %v", c.reader, c.read, granted, c.may) + } + } + bucket := func(m, n string) Bucket { return Bucket{Module: m, Name: n} } + users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": { + {Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")}, + Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, + {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, + {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, + {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, + {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: BusToolsProcess}, {Process: "other"}}}, + {Module: "shop", Processes: []ProcessPrincipal{{Process: BusToolsProcess}}}, + }}}) + if err != nil { + t.Fatal(err) + } + may := map[string][]string{ + "a.mesh-conditions": {"$KV.mesh-conditions_conditions.machine.g14.silent"}, + WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, + "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, + "a.nats.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", + "$KV.mesh-host_acts.g14.a1", "$JS.API.STREAM.INFO.KV_mesh-host_acts"}, + } + mayNot := map[string][]string{ + // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). + "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, + "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, + "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, + // The watchman never deletes or changes the controller's consumer (clause 5). + WatchmanModule: {"$KV.mesh-conditions_conditions.x", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", + "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES." + WatchmanModule}, + // The bus module's tools remove the one consumer and the acts' keys, nothing else (Limits 4); its other accounts + // remove nothing, and a process of that name in another module is no tools account. + "a.nats.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.other", "$JS.API.CONSUMER.DELETE.EVENTS.a_m", + "$KV.mesh-conditions_conditions.x", "$JS.API.STREAM.DELETE.KV_mesh-host_acts", "$JS.API.STREAM.PURGE.KV_mesh-host_acts"}, + "a.nats.other": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, + "a.nats": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, + "a.shop.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, + } + seen := map[string]bool{} + for _, u := range users { + name := u.Username() + if u.Module == WatchmanModule && u.Kind == KindModule { + name = WatchmanModule + } + perms, err := PermissionsFor(u) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + for _, subject := range may[name] { + seen[name] = true + if !MayPublish(perms, subject) { + t.Errorf("%s may not publish %s", name, subject) + } + } + for _, subject := range mayNot[name] { + seen[name] = true + if MayPublish(perms, subject) { + t.Errorf("%s may publish %s", name, subject) + } + } + } + for name := range mayNot { + if !seen[name] { + t.Errorf("no user %s was composed", name) + } + } +} diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go new file mode 100644 index 0000000..e6fc9de --- /dev/null +++ b/go/buscompose/health_grants.go @@ -0,0 +1,78 @@ +package buscompose + +import "git.novox.be/novox/mesh-sdk/go/health" + +// **The health holders' grants** (novox/hq ADR 0564 clauses 2, 5 and 8, Limits 2, 4 and 8; issue 1952 S5). +// +// - The condition keeper reads the controller's open conditions until the lease write, for what clauses 8 and 9 carry +// from them, and mesh-healers the controller's hand-act log, for S15 (Limits 2): both buckets the controller +// announces readable by nobody, so each is read by its named reader alone. The keeper, mesh-healers and the +// watchman read the controller's lease (Limits 8). Nobody reads mesh-healers' `would` (clause 8): what the healers +// would have done reaches no condition. Every observer's `findings`, the healers' `attempts`, the wants and the +// haves are read through `reads` as any announced state is; and no module writes another's state, so an observer +// can never write the keeper's buckets (clause 2). +// - The watchman makes its own consumer on DEAD_LETTER_NOTICES, by its own name, pulled, and reads whether the +// controller's consumer `controller` still stands; it may neither delete nor change that consumer (clause 5). It +// lists and shows DEAD_LETTERS, which is all it does before the lease write. +// - The bus module's tools run on a process account of their own, BusToolsProcess, which may remove only the +// consumer `controller` on DEAD_LETTER_NOTICES and keys of the node-engine's acts bucket (Limits 4), never the +// backup's account `broker`, which only reads (ADR 0235). + +// The states the composer names its readers for, beside the lease (NamedReaders). +var ( + // ControllerConditions is the controller's open conditions, read by the keeper alone (ADR 0564 clauses 8, 9). + ControllerConditions = ControllerSeat + ".conditions" + // ControllerHandActs is the controller's hand-act log, read by mesh-healers alone of the four health holders (ADR + // 0564 Limits 2). + ControllerHandActs = ControllerSeat + ".hand-acts" + // HealersWould is what mesh-healers would have done while the lease names a controller: read by nobody (clause 8). + HealersWould = health.HealersModule + "." + health.WouldState +) + +// BusToolsProcess is the process of the module holding mesh-broker whose account removes from the bus (ADR 0564 Limits +// 4): its secret is broker-nats-tools, as every process account's is broker- (ADR 0436 §2). +const BusToolsProcess = "nats-tools" + +// BusRemovalGrants is the whole authority of the bus module's tools account (ADR 0564 Limits 4, issue 1952 notes 6 and +// 7): read and delete the controller's consumer on DEAD_LETTER_NOTICES (the lease write's step, clause 5), and read and +// remove keys of the node-engine's acts bucket, which nothing else may empty when it fills. STREAM.INFO with a subject +// filter lists what a removal would take, so the tools read before they remove. A further consumer or bucket is a change +// of this grant. +func BusRemovalGrants() []string { + return []string{ + "$JS.API.CONSUMER.INFO." + DeadLetterNoticesStream + "." + ControllerName, + "$JS.API.CONSUMER.DELETE." + DeadLetterNoticesStream + "." + ControllerName, + "$JS.API.STREAM.INFO.KV_" + EngineActsBucket, + "$KV." + EngineActsBucket + ".>", + } +} + +// removesEngineActs is the bus module's tools account removing keys of the node-engine's acts bucket: the writers +// table's one exemption from a node's engine being the only writer of its keys, for removal (ADR 0564 Limits 4). +func removesEngineActs(p Principal, pattern string) bool { + return p.Kind == KindProcess && p.RemovesFromTheBus && pattern == "$KV."+EngineActsBucket+".>" +} + +// WatchmanDeadLetterConsumer is the watchman's own consumer on DEAD_LETTER_NOTICES, made only once the lease names no +// controller and `controller` is gone (ADR 0564 clause 5). +const WatchmanDeadLetterConsumer = WatchmanModule + +// WatchmanDeadLetterGrants are what the watchman may publish for the dead letters (ADR 0564 clause 5): its own pulled +// consumer on DEAD_LETTER_NOTICES (made, read, pulled, acknowledged), whether `controller` still stands (read only), and a +// read of DEAD_LETTERS: its list by subject, one kept message, and an ordered consumer of its own to page them. +func WatchmanDeadLetterGrants() []string { + own := DeadLetterNoticesStream + "." + WatchmanDeadLetterConsumer + return []string{ + "$JS.API.STREAM.INFO." + DeadLetterNoticesStream, + "$JS.API.CONSUMER.INFO." + DeadLetterNoticesStream + "." + ControllerName, + "$JS.API.CONSUMER.CREATE." + own, + "$JS.API.CONSUMER.INFO." + own, + "$JS.API.CONSUMER.MSG.NEXT." + own, + "$JS.ACK." + own + ".>", + "$JS.API.STREAM.INFO." + DeadLettersStream, + "$JS.API.STREAM.MSG.GET." + DeadLettersStream, + "$JS.API.CONSUMER.CREATE." + DeadLettersStream + ".>", + "$JS.API.CONSUMER.DELETE." + DeadLettersStream + ".*", + "$JS.FC." + DeadLettersStream + ".>", + } +} diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 74981b0..26a5fbb 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -196,6 +196,9 @@ type Principal struct { // whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing // else — not its declarations, which the node's tool runtime serves for it. SnapshotsTheBus bool + // RemovesFromTheBus is the bus module's tools process (BusToolsProcess, novox/hq ADR 0564 Limits 4): its whole + // authority is BusRemovalGrants. + RemovesFromTheBus bool // WitnessesController is a machine principal whose node-engine witnesses the controller's upgrades: // the machine runs the controller (novox/hq ADR 0236, lease/witness.go). It may read the lease's one @@ -655,6 +658,16 @@ func PermissionsFor(p Principal) (Permissions, error) { return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil } + if p.Kind == KindProcess && p.RemovesFromTheBus { + pub := BusRemovalGrants() + sort.Strings(pub) + if err := CheckWriters(p, pub); err != nil { + return Permissions{}, err + } + // Its own inbox for the answers, and nothing else (Limits 4). + return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil + } + var pub, sub []string switch p.Kind { case KindController: @@ -988,6 +1001,8 @@ func PermissionsFor(p Principal) (Permissions, error) { if p.Module == WatchmanModule { sub = append(sub, BusConnectSubject, BusDisconnectSubject) pub = append(pub, BusConnzSubject) + // And the dead letters, from its own consumer, never the controller's (novox/hq ADR 0564 clause 5). + pub = append(pub, WatchmanDeadLetterGrants()...) } // The grants part alone publishes the modules' memberships into the assignments stream (novox/hq ADR 0547 §5; // the operator, 2026-10-25 22:31): one per subject, `mesh.assignment..`, reading the last one of each diff --git a/go/buscompose/userlist_switch.go b/go/buscompose/userlist_switch.go index 6e2cb07..266eaad 100644 --- a/go/buscompose/userlist_switch.go +++ b/go/buscompose/userlist_switch.go @@ -5,6 +5,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // **The grants part's user list, sealed to the bus node** (novox/hq ADR 0547 clause 7, issue 1945). The grants part writes @@ -81,8 +82,13 @@ const VaultBusAccounts = "mesh-vault.bus-accounts" // are read by the vault alone (novox/hq ADR 0561 §2). // The controller's lease is read by the parts that turn on at its write (ADR 0561 §3, §6), and by the assigner, which // reads the catalogue's placement alone from then (ADR 0549 §5, issue 774). +// +// The health holders (novox/hq ADR 0564 Limits 2 and 8, clauses 8 and 9): the condition keeper, mesh-healers and the +// watchman read the lease; the keeper alone the controller's conditions, mesh-healers alone its hand-act log; and nobody +// the healers' `would` (health_grants.go). var NamedReaders = map[string][]string{VaultBusAccounts: {GrantsModule}, GrantsAccounts: {VaultSeat}, - ControllerLease: {VaultSeat, GrantsModule, delivery.AssignerModule}} + ControllerLease: {VaultSeat, GrantsModule, delivery.AssignerModule, health.KeeperModule, health.HealersModule, WatchmanModule}, + ControllerConditions: {health.KeeperModule}, ControllerHandActs: {health.HealersModule}, HealersWould: {}} // ReaderMay says whether module may read the state read (`.`) by the named readers: always, for a state // that names none. diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 5629510..10295ff 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -204,7 +204,7 @@ func Users(r Records) ([]Principal, error) { out = append(out, Principal{ Kind: KindProcess, Node: node, Module: d.Module, Process: pr.Process, Emits: pr.Emits, Queues: pr.Queues, State: pr.State, Uses: d.Uses, Asks: pr.Asks, - Invokes: pr.Invokes, + Invokes: pr.Invokes, RemovesFromTheBus: d.SnapshotsTheBus && pr.Process == BusToolsProcess, }) } if runtimeHere && d.Module == RuntimeModule { diff --git a/go/buscompose/writers.go b/go/buscompose/writers.go index 7649cb2..465d83d 100644 --- a/go/buscompose/writers.go +++ b/go/buscompose/writers.go @@ -249,7 +249,11 @@ var WritersTable = []WriterRow{ // node-engine's declared state, created by the grants part, written by each node's engine under its own keys. {State: "a machine's hand acts", Writer: "the node-engine of the machine, under its own keys `.>`, compare-and-set", KeptIn: "key-value " + EngineActsBucket + ", the node-engine's declared state acts", - Others: "read through the node seat's verb acts", Subjects: kvOf(EngineActsBucket), Writes: ownNodeKeys}, + Others: "read through the node seat's verb acts", Subjects: kvOf(EngineActsBucket), + Writes: func(p Principal, pattern string) bool { + return ownNodeKeys(p, pattern) || removesEngineActs(p, pattern) + }, + Shared: "the bus module's tools account removes keys of a full bucket, for removal only (novox/hq ADR 0564 Limits 4)"}, // The contents of a declaration's files over 64 KiB (novox/hq ADR 0486 §2): the controller alone writes the bucket, each // object under the digest of its content; a node-engine reads one by its name, with no list and no write. {State: "a declaration's large files", Writer: "controller, each object under the sha256 digest of its content; the " + From 0b12b62a0117b69ed9b975984ddcc7f24db7cc2e Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 11:17:45 +0100 Subject: [PATCH 10/52] A failure among the change's own tests ends the check ask's level there, so the rest of the suite is not paid for (issue 1289) Quick fix chosen (in askrun.go, the one place every build module's check ask runs): own-test packages already ran first, but the rest ran after their failure (issue 1921). 1921 still holds past them. --- go/buildstep/askrun.go | 14 +++++-- go/buildstep/askrun_test.go | 82 +++++++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 33 deletions(-) diff --git a/go/buildstep/askrun.go b/go/buildstep/askrun.go index f1ef95b..c9fa172 100644 --- a/go/buildstep/askrun.go +++ b/go/buildstep/askrun.go @@ -228,8 +228,10 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string _ = json.Unmarshal(r, &t) return t.Passed, nil } - // Every test step of the level runs, also after one that did not pass, so one check names every failing test of the - // level and not only the first (issue 1921); the level as a whole passes only when each of its steps did. + // The packages (or Node directories) whose tests the change touches run first, and a failure among them ends the level + // there, so a change's own failing test is answered without the rest of the suite (issue 1289; ADR 0444). Past them every + // test step of the level runs, also after one that did not pass, so one check names every failing test of the rest of + // the level and not only the first (issue 1921); the level passes only when each of its steps did. all := true if s.Digest == nil { // A module with no digest (Node) tests each module directory of the tree, those whose tests changed first. @@ -244,6 +246,9 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string } all = all && passed } + if !all { + return false, nil // the change's own tests failed: the others are not run (issue 1289) + } } return all, nil } @@ -279,7 +284,10 @@ func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string if err != nil { return false, fmt.Errorf("the %s level: %w", level, err) } - all = all && passed // the packages left still run (issue 1921) + all = all && passed // past the change's own packages the others still run (issue 1921) + if priority && !all { + return false, nil // the change's own tests failed: the others are not run (issue 1289) + } } return all, nil } diff --git a/go/buildstep/askrun_test.go b/go/buildstep/askrun_test.go index 49ea614..a2dbefe 100644 --- a/go/buildstep/askrun_test.go +++ b/go/buildstep/askrun_test.go @@ -113,7 +113,7 @@ func TestRunAskRefusesAnotherToolchain(t *testing.T) { } // 1887/A3, A5a: a check ask runs the levels of one check after one fetch: it digests the unit level, takes the kept -// verdicts, tests the packages whose tests changed first, then the rest also after a failure (issue 1921), and runs no +// verdicts, tests the packages whose tests changed first and, when one of them fails, no other (issue 1289), and runs no // level after one that did not pass (ADR 0484 clause 13). func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) { r := &recorder{ @@ -129,8 +129,7 @@ func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) { got := strings.Join(r.calls, "\n") want := `fetch {"base":"` + runCommit + `","commit":"` + runCommit + `","repository":"novox/x"} digest {"level":"unit","workspace":"/w"} -test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"} -test {"level":"unit","packages":["x/pkg/a"],"workspace":"/w"}` +test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}` if got != want { t.Fatalf("calls:\n%s\nwant:\n%s", got, want) } @@ -139,7 +138,7 @@ test {"level":"unit","packages":["x/pkg/a"],"workspace":"/w"}` t.Fatalf("levels %+v, want the unit level alone", a.Levels) } u := a.Levels[0] - if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 2 || u.Digest == nil { + if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 1 || u.Digest == nil { t.Fatalf("the unit level %+v", u) } @@ -254,41 +253,64 @@ func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) { } } -// Issue 1921: a failure in the packages whose tests changed does not stop the level: the other packages still run, so two -// failing packages, one changed and one not, are two test results of the one level, and the level does not pass. A Node -// check runs every module directory after one fails too. -func TestRunAskRunsEveryPackageAfterAFailure(t *testing.T) { - r := &recorder{ - fetch: map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}}, - digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b"}, - tests: map[string]bool{"x/pkg/a": false, "x/pkg/b": false}, - } - out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelIntegration, delivery.LevelUnit}, - Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}})) - if err != nil { - t.Fatal(err) - } - a := out.(delivery.CheckAnswer) - if len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 { - t.Fatalf("levels %+v, want the one level with both packages' results", a.Levels) - } - for i, raw := range a.Levels[0].Tests { - var res struct{ Passed bool } - if err := json.Unmarshal(raw, &res); err != nil || res.Passed { - t.Errorf("result %d %s, want a failure", i, raw) +// Issue 1289: a failure in the packages whose tests changed ends the level there, the others are not run, and no later level +// runs. Issue 1921: past the change's own packages every package still runs, so two failing packages none of which the +// change touched are two test results of the one level. A Node check does the same with its module directories. +func TestRunAskStopsAtTheChangesOwnFailureAndRunsTheRestAfterOthers(t *testing.T) { + for _, row := range []struct { + name string + changed []string + tests map[string]bool + want []string // the packages tested, in order + }{ + {"the change's own test fails: nothing else runs", []string{"pkg/b/b_test.go"}, map[string]bool{"x/pkg/a": false, "x/pkg/b": false}, []string{"x/pkg/b"}}, + {"the change's own test passes: the rest runs, also after a failure", []string{"pkg/b/b_test.go"}, map[string]bool{"x/pkg/a": false, "x/pkg/b": true, "x/pkg/c": false}, []string{"x/pkg/b", "x/pkg/a x/pkg/c"}}, + {"no test changed: every failure is named", nil, map[string]bool{"x/pkg/a": false, "x/pkg/b": false}, []string{"x/pkg/a x/pkg/b"}}, + } { + r := &recorder{fetch: map[string]any{"workspace": "/w", "changed": row.changed}, tests: row.tests, + digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b"}} + if _, ok := row.tests["x/pkg/c"]; ok { + r.digests["x/pkg/c"] = "d-c" + } + out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration}, + Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}})) + if err != nil { + t.Fatal(err) + } + var got []string + for _, c := range r.calls[1:] { + if strings.HasPrefix(c, "test ") { + var a struct{ Packages []string } + _ = json.Unmarshal([]byte(strings.TrimPrefix(c, "test ")), &a) + got = append(got, strings.Join(a.Packages, " ")) + } + } + // the passing row goes on to the integration level, which tests the same packages again; the first level is what is judged + if n := len(row.want); len(got) < n || fmt.Sprint(got[:n]) != fmt.Sprint(row.want) { + t.Errorf("%s: tested %v, want %v first", row.name, got, row.want) + } + a := out.(delivery.CheckAnswer) + failed := false + for _, raw := range a.Levels[0].Tests { + var res struct{ Passed bool } + _ = json.Unmarshal(raw, &res) + failed = failed || !res.Passed + } + if failed && (len(a.Levels) != 1 || len(got) != len(row.want)) { + t.Errorf("%s: a failed level went on: levels %d, tests %v", row.name, len(a.Levels), got) } } - r = &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}}, + r := &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}}, tests: map[string]bool{"a": true, "b": false}} s := r.steps() s.Digest = nil - out, err = RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration}, + out, err := RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration}, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}})) if err != nil { t.Fatal(err) } - if a := out.(delivery.CheckAnswer); len(r.calls) != 3 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 { - t.Fatalf("calls %v, levels %+v: want both directories of the unit level and no integration level", r.calls, a.Levels) + if a := out.(delivery.CheckAnswer); len(r.calls) != 2 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 1 { + t.Fatalf("calls %v, levels %+v: want the changed directory b alone and no integration level", r.calls, a.Levels) } } From 2a04bf0ac5bfcb98029ef04debb28071f7e1d26f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:43:15 +0100 Subject: [PATCH 11/52] Let the watchman read and delete a given-up message only on the streams dead letters come from, each by name and never a bucket's, so keeping dead letters cannot read every bucket (issue 1952 S5) --- go/buscompose/buscompose_test.go | 19 ++++++-- go/buscompose/health_grants.go | 75 +++++++++++++++++++++++++++++--- go/buscompose/nats.go | 5 ++- go/buscompose/users.go | 3 ++ go/buscompose/writers.go | 3 +- 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 4d23e3f..5e7f36e 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -39,7 +39,7 @@ var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATE func beyondTheControllers(user string) map[string]bool { out := map[string]bool{} if strings.HasSuffix(user, "."+WatchmanModule) { - for _, s := range WatchmanDeadLetterGrants() { + for _, s := range WatchmanDeadLetterGrants(nil) { out[s] = true } } @@ -593,6 +593,8 @@ func healthChecks(t *testing.T) { {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, + {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, + Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: BusToolsProcess}, {Process: "other"}}}, {Module: "shop", Processes: []ProcessPrincipal{{Process: BusToolsProcess}}}, }}}) @@ -603,6 +605,8 @@ func healthChecks(t *testing.T) { "a.mesh-conditions": {"$KV.mesh-conditions_conditions.machine.g14.silent"}, WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, + WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.MSG.DELETE.SEAT_TELEGRAM_SENDER", + "$JS.API.STREAM.MSG.GET.M_WORK", "$JS.API.STREAM.MSG.DELETE.DEAD_LETTERS", "mesh.events.dead.EVENTS.a_m", "mesh.again.a_m.x"}, "a.nats.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1", "$JS.API.STREAM.INFO.KV_mesh-host_acts"}, } @@ -612,6 +616,10 @@ func healthChecks(t *testing.T) { "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, // The watchman never deletes or changes the controller's consumer (clause 5). + // Its reads and deletes of a message are on the streams a dead letter comes from, named, never a bucket's + // (the lead, 2026-10-29). + WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.KV_mesh-vault_bus-accounts", "$JS.API.STREAM.MSG.GET.KV_x", + "$JS.API.STREAM.MSG.DELETE.KV_mesh-host_acts", "$JS.API.STREAM.MSG.GET.OBJ_mesh-grants-user-list", "$JS.API.STREAM.MSG.GET.OTHER"}, WatchmanModule: {"$KV.mesh-conditions_conditions.x", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES." + WatchmanModule}, // The bus module's tools remove the one consumer and the acts' keys, nothing else (Limits 4); its other accounts @@ -632,13 +640,18 @@ func healthChecks(t *testing.T) { if err != nil { t.Fatalf("%s: %v", name, err) } - for _, subject := range may[name] { + subjects := append(append([]string(nil), may[name]...), may[name+" sources"]...) + denied := append(append([]string(nil), mayNot[name]...), mayNot[name+" sources"]...) + if len(may[name+" sources"]) > 0 { + seen[name+" sources"] = true + } + for _, subject := range subjects { seen[name] = true if !MayPublish(perms, subject) { t.Errorf("%s may not publish %s", name, subject) } } - for _, subject := range mayNot[name] { + for _, subject := range denied { seen[name] = true if MayPublish(perms, subject) { t.Errorf("%s may publish %s", name, subject) diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go index e6fc9de..46605dc 100644 --- a/go/buscompose/health_grants.go +++ b/go/buscompose/health_grants.go @@ -1,6 +1,11 @@ package buscompose -import "git.novox.be/novox/mesh-sdk/go/health" +import ( + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/health" +) // **The health holders' grants** (novox/hq ADR 0564 clauses 2, 5 and 8, Limits 2, 4 and 8; issue 1952 S5). // @@ -58,11 +63,16 @@ func removesEngineActs(p Principal, pattern string) bool { const WatchmanDeadLetterConsumer = WatchmanModule // WatchmanDeadLetterGrants are what the watchman may publish for the dead letters (ADR 0564 clause 5): its own pulled -// consumer on DEAD_LETTER_NOTICES (made, read, pulled, acknowledged), whether `controller` still stands (read only), and a -// read of DEAD_LETTERS: its list by subject, one kept message, and an ordered consumer of its own to page them. -func WatchmanDeadLetterGrants() []string { +// consumer on DEAD_LETTER_NOTICES (made, read, pulled, acknowledged), whether `controller` still stands (read only), a +// read of DEAD_LETTERS (its list by subject, one kept message, an ordered consumer of its own to page them, a drop), the +// copies kept and the messages delivered again (`mesh.events.dead.>`, `mesh.again.>`), and on each stream a notice can +// name (sources: EVENTS, the seats' work queues and the modules' own) the message given up on read, its consumer read, +// and its message deleted from the work queue when delivered again, each stream by its name. Never a wildcard over the +// streams and never a bucket's or an object store's stream: a read of any message would read every bucket (the lead, +// 2026-10-29, on issue 1952 S5). +func WatchmanDeadLetterGrants(sources []string) []string { own := DeadLetterNoticesStream + "." + WatchmanDeadLetterConsumer - return []string{ + out := []string{ "$JS.API.STREAM.INFO." + DeadLetterNoticesStream, "$JS.API.CONSUMER.INFO." + DeadLetterNoticesStream + "." + ControllerName, "$JS.API.CONSUMER.CREATE." + own, @@ -71,8 +81,63 @@ func WatchmanDeadLetterGrants() []string { "$JS.ACK." + own + ".>", "$JS.API.STREAM.INFO." + DeadLettersStream, "$JS.API.STREAM.MSG.GET." + DeadLettersStream, + "$JS.API.STREAM.MSG.DELETE." + DeadLettersStream, "$JS.API.CONSUMER.CREATE." + DeadLettersStream + ".>", "$JS.API.CONSUMER.DELETE." + DeadLettersStream + ".*", "$JS.FC." + DeadLettersStream + ".>", + deadLetterPrefix + ">", + againPrefix + ">", } + for _, s := range DeadLetterSources(sources) { + out = append(out, "$JS.API.STREAM.MSG.GET."+s, "$JS.API.STREAM.MSG.DELETE."+s, "$JS.API.CONSUMER.INFO."+s+".*") + } + return out +} + +// DeadLetterSources are the streams of names a dead letter may come from, sorted, none twice: each a plain name, never a +// bucket's (KV_) or an object store's (OBJ_) stream, the dead letters' own two streams, or a pattern. +func DeadLetterSources(names []string) []string { + seen := map[string]bool{} + var out []string + for _, n := range names { + switch { + case seen[n], !safeSubject.MatchString(n), strings.HasPrefix(n, "KV_"), strings.HasPrefix(n, "OBJ_"), + n == DeadLettersStream, n == DeadLetterNoticesStream: + continue + } + seen[n] = true + out = append(out, n) + } + sort.Strings(out) + return out +} + +// deadLetterSourcesOf are the streams the records' consumers give up on: EVENTS, the work queue of every seat that +// accepts work, and every module's and process's own work queue. +func deadLetterSourcesOf(r Records) []string { + out := []string{EventsStream} + for _, mods := range r.Assigned { + for _, d := range mods { + for _, s := range append(append([]Seat(nil), d.Holds...), d.Uses...) { + if len(s.Accepts) > 0 { + out = append(out, seatStreamName(s.Name)) + } + } + for _, q := range d.Queues { + out = append(out, q.Stream) + } + for _, pr := range d.Processes { + for _, q := range pr.Queues { + out = append(out, q.Stream) + } + } + } + } + return DeadLetterSources(out) +} + +// keepsDeadLetters is the watchman keeping the dead letters: a second writer of the copies and the deliveries again +// beside the controller, which acts until its lease is handed away and the watchman after (ADR 0564 clause 5). +func keepsDeadLetters(p Principal, pattern string) bool { + return isController(p, pattern) || (p.Kind == KindModule && p.Module == WatchmanModule) } diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 26a5fbb..e27d05d 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -199,6 +199,9 @@ type Principal struct { // RemovesFromTheBus is the bus module's tools process (BusToolsProcess, novox/hq ADR 0564 Limits 4): its whole // authority is BusRemovalGrants. RemovesFromTheBus bool + // DeadLetterSources are, for the watchman, the streams a dead letter it keeps may come from, by name + // (deadLetterSourcesOf, novox/hq ADR 0564 clause 5): never a bucket's. + DeadLetterSources []string // WitnessesController is a machine principal whose node-engine witnesses the controller's upgrades: // the machine runs the controller (novox/hq ADR 0236, lease/witness.go). It may read the lease's one @@ -1002,7 +1005,7 @@ func PermissionsFor(p Principal) (Permissions, error) { sub = append(sub, BusConnectSubject, BusDisconnectSubject) pub = append(pub, BusConnzSubject) // And the dead letters, from its own consumer, never the controller's (novox/hq ADR 0564 clause 5). - pub = append(pub, WatchmanDeadLetterGrants()...) + pub = append(pub, WatchmanDeadLetterGrants(p.DeadLetterSources)...) } // The grants part alone publishes the modules' memberships into the assignments stream (novox/hq ADR 0547 §5; // the operator, 2026-10-25 22:31): one per subject, `mesh.assignment..`, reading the last one of each diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 10295ff..2d1fc8c 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -229,6 +229,9 @@ func Users(r Records) ([]Principal, error) { }) // **The grants part creates what a manifest declares** (novox/hq ADR 0547 §5; the operator, 2026-10-26 // 08:54): every placed module's buckets, logs and streams, each by its stream's name. + if d.Module == WatchmanModule { + out[len(out)-1].DeadLetterSources = deadLetterSourcesOf(r) + } if d.Module == GrantsModule { out[len(out)-1].CreatesStreams = declaredStreamsOf(r) grants = len(out) - 1 diff --git a/go/buscompose/writers.go b/go/buscompose/writers.go index 465d83d..0911597 100644 --- a/go/buscompose/writers.go +++ b/go/buscompose/writers.go @@ -220,7 +220,8 @@ var WritersTable = []WriterRow{ // and delivered again or dropped only through its verb, with why. {State: "a message a consumer gave up on", Writer: "controller", KeptIn: "the bus, the stream " + DeadLettersStream, Others: "read, delivered again or dropped through the controller's dead-letters verb", - Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: isController}, + Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: keepsDeadLetters, + Shared: "the watchman keeps them from the controller's lease write, the controller before it (novox/hq ADR 0564 clause 5)"}, // The record mirrors (novox/hq ADR 0289 §9, design 54 §2): rows of this table, so a grant that lets // anything else write them is refused at composition. Built so far: the assignments and each registered // module's declared config (design 57 F3), and the list of predefined keys kept (design 57 K3). From 69fcb6c1c1ba5b6e1f78fd148b99083cd55649d1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:44:53 +0100 Subject: [PATCH 12/52] Fit the watchman's grant to what L4 does: hear its watchdogs' words and read a source's age and message, never deliver again or drop, so it holds no more than its job (issue 1952 S5) --- go/buscompose/buscompose_test.go | 25 ++++++++++++----- go/buscompose/health_grants.go | 46 ++++++++++++++++++++------------ go/buscompose/nats.go | 2 ++ go/buscompose/writers.go | 2 +- 4 files changed, 51 insertions(+), 24 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 5e7f36e..4a10d97 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -34,12 +34,13 @@ var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATE // keep applying without composing their own until the next take-over. Each principal's hearing of its buckets' creation // (issue 1790) is the other row beyond, matched by createdAdvisory. // -// And the watchman's dead letters (novox/hq ADR 0564 clause 5, issue 1952 S5): its own consumer on DEAD_LETTER_NOTICES and -// a read of DEAD_LETTERS, the controller's job until the lease write. +// And the watchman's dead letters and watchdogs (novox/hq ADR 0564 clauses 4, 5, issue 1952 S5): its own consumer on +// DEAD_LETTER_NOTICES, a read of DEAD_LETTERS, and the words its watchdogs hear, the controller's jobs until the lease +// write. func beyondTheControllers(user string) map[string]bool { out := map[string]bool{} if strings.HasSuffix(user, "."+WatchmanModule) { - for _, s := range WatchmanDeadLetterGrants(nil) { + for _, s := range append(WatchmanDeadLetterGrants(nil), WatchmanHears()...) { out[s] = true } } @@ -605,8 +606,8 @@ func healthChecks(t *testing.T) { "a.mesh-conditions": {"$KV.mesh-conditions_conditions.machine.g14.silent"}, WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, - WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.MSG.DELETE.SEAT_TELEGRAM_SENDER", - "$JS.API.STREAM.MSG.GET.M_WORK", "$JS.API.STREAM.MSG.DELETE.DEAD_LETTERS", "mesh.events.dead.EVENTS.a_m", "mesh.again.a_m.x"}, + WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.INFO.SEAT_TELEGRAM_SENDER", + "$JS.API.STREAM.MSG.GET.M_WORK", "$JS.API.STREAM.INFO.EVENTS", "mesh.events.dead.EVENTS.a_m"}, "a.nats.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1", "$JS.API.STREAM.INFO.KV_mesh-host_acts"}, } @@ -619,7 +620,10 @@ func healthChecks(t *testing.T) { // Its reads and deletes of a message are on the streams a dead letter comes from, named, never a bucket's // (the lead, 2026-10-29). WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.KV_mesh-vault_bus-accounts", "$JS.API.STREAM.MSG.GET.KV_x", - "$JS.API.STREAM.MSG.DELETE.KV_mesh-host_acts", "$JS.API.STREAM.MSG.GET.OBJ_mesh-grants-user-list", "$JS.API.STREAM.MSG.GET.OTHER"}, + "$JS.API.STREAM.MSG.DELETE.KV_mesh-host_acts", "$JS.API.STREAM.MSG.GET.OBJ_mesh-grants-user-list", "$JS.API.STREAM.MSG.GET.OTHER", + // In L4 it neither delivers again nor drops (clause 5, the lead 2026-10-29). + "$JS.API.STREAM.MSG.DELETE.EVENTS", "$JS.API.STREAM.MSG.DELETE.DEAD_LETTERS", "$JS.API.CONSUMER.CREATE.DEAD_LETTERS.x", + "mesh.again.a_m.x"}, WatchmanModule: {"$KV.mesh-conditions_conditions.x", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES." + WatchmanModule}, // The bus module's tools remove the one consumer and the acts' keys, nothing else (Limits 4); its other accounts @@ -640,6 +644,15 @@ func healthChecks(t *testing.T) { if err != nil { t.Fatalf("%s: %v", name, err) } + if name == WatchmanModule { + // It hears its watchdogs' words on core subjects (Limits 3, 11). + for _, subject := range []string{"mesh.mod.power.event.sleeping", "mesh.control.g14.tools-alive", + "mesh.seat.mesh-self-check.event.checked", "mesh.mod.keycloak.event.provisioner.failing"} { + if !MaySubscribe(perms, subject) { + t.Errorf("the watchman may not hear %s", subject) + } + } + } subjects := append(append([]string(nil), may[name]...), may[name+" sources"]...) denied := append(append([]string(nil), mayNot[name]...), mayNot[name+" sources"]...) if len(may[name+" sources"]) > 0 { diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go index 46605dc..26f3c3b 100644 --- a/go/buscompose/health_grants.go +++ b/go/buscompose/health_grants.go @@ -62,14 +62,14 @@ func removesEngineActs(p Principal, pattern string) bool { // controller and `controller` is gone (ADR 0564 clause 5). const WatchmanDeadLetterConsumer = WatchmanModule -// WatchmanDeadLetterGrants are what the watchman may publish for the dead letters (ADR 0564 clause 5): its own pulled -// consumer on DEAD_LETTER_NOTICES (made, read, pulled, acknowledged), whether `controller` still stands (read only), a -// read of DEAD_LETTERS (its list by subject, one kept message, an ordered consumer of its own to page them, a drop), the -// copies kept and the messages delivered again (`mesh.events.dead.>`, `mesh.again.>`), and on each stream a notice can -// name (sources: EVENTS, the seats' work queues and the modules' own) the message given up on read, its consumer read, -// and its message deleted from the work queue when delivered again, each stream by its name. Never a wildcard over the -// streams and never a bucket's or an object store's stream: a read of any message would read every bucket (the lead, -// 2026-10-29, on issue 1952 S5). +// WatchmanDeadLetterGrants are what the watchman may publish for the dead letters in L4 (ADR 0564 clause 5), at least +// privilege: its own pulled consumer on DEAD_LETTER_NOTICES (made, read, pulled, acknowledged); whether `controller` +// still stands (read only); a read of DEAD_LETTERS (its information and one kept message); a copy kept +// (`mesh.events.dead.>`); and on each stream a notice can name (sources: EVENTS, the seats' work queues and the +// modules' own), its information, for a source's age (ADR 0376), and the message given up on, each stream by its name. +// Never a wildcard over the streams and never a bucket's or an object store's stream: a read of any message would read +// every bucket (the lead, 2026-10-29, on issue 1952 S5). Delivering again and dropping stay refused in L4, so it may +// neither delete a message nor publish `mesh.again.>`. func WatchmanDeadLetterGrants(sources []string) []string { own := DeadLetterNoticesStream + "." + WatchmanDeadLetterConsumer out := []string{ @@ -81,19 +81,29 @@ func WatchmanDeadLetterGrants(sources []string) []string { "$JS.ACK." + own + ".>", "$JS.API.STREAM.INFO." + DeadLettersStream, "$JS.API.STREAM.MSG.GET." + DeadLettersStream, - "$JS.API.STREAM.MSG.DELETE." + DeadLettersStream, - "$JS.API.CONSUMER.CREATE." + DeadLettersStream + ".>", - "$JS.API.CONSUMER.DELETE." + DeadLettersStream + ".*", - "$JS.FC." + DeadLettersStream + ".>", deadLetterPrefix + ">", - againPrefix + ">", } for _, s := range DeadLetterSources(sources) { - out = append(out, "$JS.API.STREAM.MSG.GET."+s, "$JS.API.STREAM.MSG.DELETE."+s, "$JS.API.CONSUMER.INFO."+s+".*") + out = append(out, "$JS.API.STREAM.INFO."+s, "$JS.API.STREAM.MSG.GET."+s) } return out } +// WatchmanHears are the core subjects the watchman subscribes, with no durable consumer (ADR 0564 clauses 4, 5 and +// Limits 3, 11): the power seat's words (S1, ADR 0211), the providers' failing and recovered words (S8), each tool +// runner's beat (S11), the self-check's checked (S10), a consumer deleted (S9), and the dead letters kept. +func WatchmanHears() []string { + return []string{ + "mesh.mod.power.event.*", + "mesh.mod.*.event.provisioner.failing", + "mesh.mod.*.event.provisioner.recovered", + "mesh.control.*.tools-alive", + "mesh.seat.mesh-self-check.event." + health.EventChecked, + "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", + deadLetterPrefix + ">", + } +} + // DeadLetterSources are the streams of names a dead letter may come from, sorted, none twice: each a plain name, never a // bucket's (KV_) or an object store's (OBJ_) stream, the dead letters' own two streams, or a pattern. func DeadLetterSources(names []string) []string { @@ -136,8 +146,10 @@ func deadLetterSourcesOf(r Records) []string { return DeadLetterSources(out) } -// keepsDeadLetters is the watchman keeping the dead letters: a second writer of the copies and the deliveries again -// beside the controller, which acts until its lease is handed away and the watchman after (ADR 0564 clause 5). +// keepsDeadLetters is the watchman keeping the dead letters: a second writer of the copies beside the controller, which +// keeps them until its lease is handed away and the watchman after (ADR 0564 clause 5). Delivering again stays the +// controller's in L4. func keepsDeadLetters(p Principal, pattern string) bool { - return isController(p, pattern) || (p.Kind == KindModule && p.Module == WatchmanModule) + return isController(p, pattern) || + (p.Kind == KindModule && p.Module == WatchmanModule && pattern == deadLetterPrefix+">") } diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index e27d05d..25edb1a 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -1006,6 +1006,8 @@ func PermissionsFor(p Principal) (Permissions, error) { pub = append(pub, BusConnzSubject) // And the dead letters, from its own consumer, never the controller's (novox/hq ADR 0564 clause 5). pub = append(pub, WatchmanDeadLetterGrants(p.DeadLetterSources)...) + // And the words its watchdogs hear (ADR 0564 clause 4, Limits 3, 11). + sub = append(sub, WatchmanHears()...) } // The grants part alone publishes the modules' memberships into the assignments stream (novox/hq ADR 0547 §5; // the operator, 2026-10-25 22:31): one per subject, `mesh.assignment..`, reading the last one of each diff --git a/go/buscompose/writers.go b/go/buscompose/writers.go index 0911597..1d94184 100644 --- a/go/buscompose/writers.go +++ b/go/buscompose/writers.go @@ -221,7 +221,7 @@ var WritersTable = []WriterRow{ {State: "a message a consumer gave up on", Writer: "controller", KeptIn: "the bus, the stream " + DeadLettersStream, Others: "read, delivered again or dropped through the controller's dead-letters verb", Subjects: []string{deadLetterPrefix + ">", againPrefix + ">"}, Writes: keepsDeadLetters, - Shared: "the watchman keeps them from the controller's lease write, the controller before it (novox/hq ADR 0564 clause 5)"}, + Shared: "the watchman keeps the copies from the controller's lease write, the controller before it; delivering again stays the controller's (novox/hq ADR 0564 clause 5)"}, // The record mirrors (novox/hq ADR 0289 §9, design 54 §2): rows of this table, so a grant that lets // anything else write them is refused at composition. Built so far: the assignments and each registered // module's declared config (design 57 F3), and the list of predefined keys kept (design 57 K3). From 7000bd5049ec9d7cefd8ec0225648670cb0ac49c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:52:52 +0100 Subject: [PATCH 13/52] Let a tools bundle declare its own bus account and what it may remove, so the bus tools remove the controller's dead-letter consumer and the node-engine's acts on a declared account with no idle service (issue 1952 S5, ADR 0564 Limits 4) --- go/buscompose/buscompose_test.go | 17 +++-- go/buscompose/health_grants.go | 49 ++++++++------ go/buscompose/nats.go | 13 ++-- go/buscompose/users.go | 5 +- go/catalogue/build.go | 2 +- go/catalogue/manifest.go | 6 ++ go/catalogue/process_bus.go | 106 ++++++++++++++++++++++++++++++- go/catalogue/process_bus_test.go | 43 +++++++++++++ 8 files changed, 208 insertions(+), 33 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 4a10d97..e4f8212 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -13,6 +13,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go/jetstream" ) @@ -596,8 +597,9 @@ func healthChecks(t *testing.T) { {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, - {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: BusToolsProcess}, {Process: "other"}}}, - {Module: "shop", Processes: []ProcessPrincipal{{Process: BusToolsProcess}}}, + {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools", + Removes: []string{catalogue.RemovesControllerDeadLetterConsumer, catalogue.RemovesEngineActs}}, {Process: "other"}}}, + {Module: "shop", Processes: []ProcessPrincipal{{Process: "nats-tools", Removes: []string{catalogue.RemovesEngineActs}}}}, }}}) if err != nil { t.Fatal(err) @@ -627,12 +629,19 @@ func healthChecks(t *testing.T) { WatchmanModule: {"$KV.mesh-conditions_conditions.x", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES.controller", "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES." + WatchmanModule}, // The bus module's tools remove the one consumer and the acts' keys, nothing else (Limits 4); its other accounts - // remove nothing, and a process of that name in another module is no tools account. + // remove nothing; an account is granted the removals its bus declares and no other (the catalogue refuses them + // outside the bus module). "a.nats.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.other", "$JS.API.CONSUMER.DELETE.EVENTS.a_m", "$KV.mesh-conditions_conditions.x", "$JS.API.STREAM.DELETE.KV_mesh-host_acts", "$JS.API.STREAM.PURGE.KV_mesh-host_acts"}, "a.nats.other": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, "a.nats": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, - "a.shop.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, + "a.shop.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-conditions_conditions.x"}, + } + // The tools account's secret is the module's own broker-, which its manifest's own-secrets names and the + // vault keeps by that name (AccountSecret). + if w := (WantedAccount{User: "a.nats.nats-tools", Kind: KindProcess, Node: "a", Module: "nats", Process: "nats-tools", + URL: "nats://a:4222"}); w.Valid() != nil || w.OwnSecret() != "broker-nats-tools" { + t.Errorf("the tools account is held in %q (%v)", w.OwnSecret(), w.Valid()) } seen := map[string]bool{} for _, u := range users { diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go index 26f3c3b..e795a85 100644 --- a/go/buscompose/health_grants.go +++ b/go/buscompose/health_grants.go @@ -1,9 +1,12 @@ package buscompose import ( + "fmt" + "slices" "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -19,9 +22,9 @@ import ( // - The watchman makes its own consumer on DEAD_LETTER_NOTICES, by its own name, pulled, and reads whether the // controller's consumer `controller` still stands; it may neither delete nor change that consumer (clause 5). It // lists and shows DEAD_LETTERS, which is all it does before the lease write. -// - The bus module's tools run on a process account of their own, BusToolsProcess, which may remove only the -// consumer `controller` on DEAD_LETTER_NOTICES and keys of the node-engine's acts bucket (Limits 4), never the -// backup's account `broker`, which only reads (ADR 0235). +// - The bus module's tools run on an account of their own, declared by their bundle's `bus` with `removes` +// (catalogue.BusAccounts), which may remove only the consumer `controller` on DEAD_LETTER_NOTICES and keys of the +// node-engine's acts bucket (Limits 4), never the backup's account `broker`, which only reads (ADR 0235). // The states the composer names its readers for, beside the lease (NamedReaders). var ( @@ -34,28 +37,32 @@ var ( HealersWould = health.HealersModule + "." + health.WouldState ) -// BusToolsProcess is the process of the module holding mesh-broker whose account removes from the bus (ADR 0564 Limits -// 4): its secret is broker-nats-tools, as every process account's is broker- (ADR 0436 §2). -const BusToolsProcess = "nats-tools" - -// BusRemovalGrants is the whole authority of the bus module's tools account (ADR 0564 Limits 4, issue 1952 notes 6 and -// 7): read and delete the controller's consumer on DEAD_LETTER_NOTICES (the lease write's step, clause 5), and read and -// remove keys of the node-engine's acts bucket, which nothing else may empty when it fills. STREAM.INFO with a subject -// filter lists what a removal would take, so the tools read before they remove. A further consumer or bucket is a change -// of this grant. -func BusRemovalGrants() []string { - return []string{ - "$JS.API.CONSUMER.INFO." + DeadLetterNoticesStream + "." + ControllerName, - "$JS.API.CONSUMER.DELETE." + DeadLetterNoticesStream + "." + ControllerName, - "$JS.API.STREAM.INFO.KV_" + EngineActsBucket, - "$KV." + EngineActsBucket + ".>", +// BusRemovalGrants are the whole authority of an account whose bus declares removals (ADR 0564 Limits 4, issue 1952 +// notes 6 and 7), each removal its grant: the controller's consumer on DEAD_LETTER_NOTICES read and deleted (the lease +// write's step, clause 5); the node-engine's acts bucket read (STREAM.INFO with a subject filter lists what a removal +// would take, so the tools read before they remove) and its keys removed, which nothing else may empty when it fills. A +// removal outside the set is refused, as the catalogue refuses it (catalogue.BusRemovals). +func BusRemovalGrants(removes []string) ([]string, error) { + var out []string + for _, r := range removes { + switch r { + case catalogue.RemovesControllerDeadLetterConsumer: + out = append(out, "$JS.API.CONSUMER.INFO."+DeadLetterNoticesStream+"."+ControllerName, + "$JS.API.CONSUMER.DELETE."+DeadLetterNoticesStream+"."+ControllerName) + case catalogue.RemovesEngineActs: + out = append(out, "$JS.API.STREAM.INFO.KV_"+EngineActsBucket, "$KV."+EngineActsBucket+".>") + default: + return nil, fmt.Errorf("a bus account removes %q, which is not one of %s (novox/hq ADR 0564 Limits 4)", r, + strings.Join(catalogue.BusRemovals, ", ")) + } } + return out, nil } -// removesEngineActs is the bus module's tools account removing keys of the node-engine's acts bucket: the writers -// table's one exemption from a node's engine being the only writer of its keys, for removal (ADR 0564 Limits 4). +// removesEngineActs is an account declared to remove keys of the node-engine's acts bucket: the writers table's one +// exemption from a node's engine being the only writer of its keys, for removal (ADR 0564 Limits 4). func removesEngineActs(p Principal, pattern string) bool { - return p.Kind == KindProcess && p.RemovesFromTheBus && pattern == "$KV."+EngineActsBucket+".>" + return p.Kind == KindProcess && slices.Contains(p.Removes, catalogue.RemovesEngineActs) && pattern == "$KV."+EngineActsBucket+".>" } // WatchmanDeadLetterConsumer is the watchman's own consumer on DEAD_LETTER_NOTICES, made only once the lease names no diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 25edb1a..83378f1 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -196,9 +196,9 @@ type Principal struct { // whole authority is BusSnapshotGrants: it copies the streams for the night's backup and nothing // else — not its declarations, which the node's tool runtime serves for it. SnapshotsTheBus bool - // RemovesFromTheBus is the bus module's tools process (BusToolsProcess, novox/hq ADR 0564 Limits 4): its whole - // authority is BusRemovalGrants. - RemovesFromTheBus bool + // Removes are, for a process or tools bundle account whose bus declares them, what it may remove from the bus + // (novox/hq ADR 0564 Limits 4): its whole authority is then BusRemovalGrants of them. + Removes []string // DeadLetterSources are, for the watchman, the streams a dead letter it keeps may come from, by name // (deadLetterSourcesOf, novox/hq ADR 0564 clause 5): never a bucket's. DeadLetterSources []string @@ -661,8 +661,11 @@ func PermissionsFor(p Principal) (Permissions, error) { return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil } - if p.Kind == KindProcess && p.RemovesFromTheBus { - pub := BusRemovalGrants() + if p.Kind == KindProcess && len(p.Removes) > 0 { + pub, err := BusRemovalGrants(p.Removes) + if err != nil { + return Permissions{}, err + } sort.Strings(pub) if err := CheckWriters(p, pub); err != nil { return Permissions{}, err diff --git a/go/buscompose/users.go b/go/buscompose/users.go index 2d1fc8c..ca23cf5 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -105,6 +105,9 @@ type ProcessPrincipal struct { Emits []string // Queues are the module's own work-queue streams this process is the one consumer of. Queues []QueueOwner + // Removes are what its account may remove from the bus, as its bus declares them (catalogue.BusRemovals; novox/hq + // ADR 0564 Limits 4): a tools bundle's account, whose whole authority is then BusRemovalGrants. + Removes []string // State is the module's own state, by local name, this process reads and writes. State []string // Asks says whether this process reads the records of its module's own asks (novox/hq ADR 0259 §3), on @@ -204,7 +207,7 @@ func Users(r Records) ([]Principal, error) { out = append(out, Principal{ Kind: KindProcess, Node: node, Module: d.Module, Process: pr.Process, Emits: pr.Emits, Queues: pr.Queues, State: pr.State, Uses: d.Uses, Asks: pr.Asks, - Invokes: pr.Invokes, RemovesFromTheBus: d.SnapshotsTheBus && pr.Process == BusToolsProcess, + Invokes: pr.Invokes, Removes: pr.Removes, }) } if runtimeHere && d.Module == RuntimeModule { diff --git a/go/catalogue/build.go b/go/catalogue/build.go index 0a2ff73..7c2a4b6 100644 --- a/go/catalogue/build.go +++ b/go/catalogue/build.go @@ -136,7 +136,7 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest, Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...), Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers), - Binary: BinaryOf(a), + Binary: BinaryOf(a), Bus: a.Bus, }) } sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name }) diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index b40dbad..ca3eba9 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -1011,6 +1011,8 @@ type Bundle struct { // Binary is the executable a bundle compiled to a binary is, at its root (novox/hq ADR 0193): // what runs it, where an interpreted bundle names an interpreter and an entrypoint. Binary string `json:"binary,omitempty"` + // Bus is the artifact's bus account, kept on the resolved manifest so the grant composition reads it (BusAccounts). + Bus *ProcessBus `json:"bus,omitempty"` } // Build says how to produce this module's artifacts from its source. @@ -1148,6 +1150,10 @@ type Artifact struct { // delivered version. A bundle that carried the package's name was delivered correctly, reported // success, and was invisible to the launcher (novox/hq 04-ISSUES/142). Binary string `json:"binary,omitempty"` + // Bus is a tools bundle's own bus account (novox/hq ADR 0564 Limits 4; the lead, 2026-10-29): read as a process's + // (ProcessBus), a principal named after the bundle's binary (BusAccounts), its secret broker-. A tools bundle + // is run by the node's tool runtime per call, so it declares its account here and runs no idle service for it. + Bus *ProcessBus `json:"bus,omitempty"` // URL is where an archive's upstream file is fetched from, for an archive that is a release // somebody else published rather than a directory of this repository (novox/hq ADR 0316 §2, §6). diff --git a/go/catalogue/process_bus.go b/go/catalogue/process_bus.go index 445fc68..c48f11c 100644 --- a/go/catalogue/process_bus.go +++ b/go/catalogue/process_bus.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "slices" "strings" ) @@ -29,6 +30,109 @@ type ProcessBus struct { // Invokes are the tools and seat verbs this process calls, each . or a role's // seat:. (as Manifest.Invokes), or * for every tool on the mesh. Invokes []string `json:"invokes,omitempty"` + // Removes are what a tools bundle's account may remove from the bus, from a closed set (BusRemovals; novox/hq ADR + // 0564 Limits 4): declared alone, by the module holding mesh-broker, never by a process. + Removes []string `json:"removes,omitempty"` +} + +// What a bus account may remove (novox/hq ADR 0564 Limits 4), each its own grant: the controller's consumer on +// DEAD_LETTER_NOTICES (the lease write's step, clause 5), and keys of the node-engine's acts bucket. A further consumer +// or bucket is a change of this set, through a decision. +const ( + RemovesControllerDeadLetterConsumer = "DEAD_LETTER_NOTICES.controller" + RemovesEngineActs = "mesh-host_acts" +) + +// BusRemovals is the closed set of what a bus account may remove. +var BusRemovals = []string{RemovesControllerDeadLetterConsumer, RemovesEngineActs} + +// BusAccount is one account of a module's own beside its `broker`: a process's (ADR 0436) or a tools bundle's (ADR 0564 +// Limits 4), by its id (the process resource's, or the bundle's binary), with what its bus says. +type BusAccount struct { + ID string + Bus ProcessBus + // Tools says it is a tools bundle's, run by the node's tool runtime, not a process the node runs. + Tools bool +} + +// BusAccounts is every own bus account of the module, processes first in resource order, then tools bundles in artifact +// order: what the grant composition composes a principal for, each delivered in its own secret broker-. A tools +// bundle's is read from the repository's build, or from the resolved manifest's bundles. +func (m Manifest) BusAccounts() []BusAccount { + var out []BusAccount + buses := m.ProcessBuses() + for _, r := range m.Resources { + id := fmt.Sprint(r["id"]) + if b, ok := buses[id]; ok && fmt.Sprint(r["type"]) == "process" { + out = append(out, BusAccount{ID: id, Bus: b}) + } + } + seen := map[string]bool{} + if m.Build != nil { + for _, a := range m.Build.Artifacts { + if a.Bus != nil && a.Kind == ArtifactBundle && BinaryOf(a) != "" && !seen[BinaryOf(a)] { + seen[BinaryOf(a)] = true + out = append(out, BusAccount{ID: BinaryOf(a), Bus: *a.Bus, Tools: true}) + } + } + } + for _, b := range m.Bundles { + if b.Bus != nil && b.Binary != "" && !seen[b.Binary] { + seen[b.Binary] = true + out = append(out, BusAccount{ID: b.Binary, Bus: *b.Bus, Tools: true}) + } + } + return out +} + +// toolsBusProblems is what is wrong with a tools bundle's bus (ADR 0564 Limits 4): a bus on anything but a bundle +// compiled to a binary the runtime loads, a removal outside the closed set, or a removal beside any other grant or by a +// module that does not hold mesh-broker. Every other field is judged as a process's. +func toolsBusProblems(m Manifest) []string { + var problems []string + var artifacts []Artifact + if m.Build != nil { + artifacts = m.Build.Artifacts + } + claimsBroker := false + for _, c := range m.Claims { + claimsBroker = claimsBroker || c.Name == "mesh-broker" + } + for _, a := range artifacts { + if a.Bus == nil { + continue + } + if a.Kind != ArtifactBundle || BinaryOf(a) == "" { + problems = append(problems, fmt.Sprintf("%s's artifact %s declares bus, and only a tools bundle compiled to a "+ + "binary has an account of its own (novox/hq ADR 0564 Limits 4)", m.Module, a.Name)) + continue + } + b := *a.Bus + if len(b.Queues) > 0 || b.Asks { + problems = append(problems, fmt.Sprintf("%s's tools bundle %s's bus consumes a queue or reads asks; a tools "+ + "bundle runs per call and holds no consumer (novox/hq ADR 0564 Limits 4)", m.Module, a.Name)) + } + for _, r := range b.Removes { + if !slices.Contains(BusRemovals, r) { + problems = append(problems, fmt.Sprintf("%s's tools bundle %s's bus removes %q, which is not one of %s "+ + "(novox/hq ADR 0564 Limits 4)", m.Module, a.Name, r, strings.Join(BusRemovals, ", "))) + } + } + if len(b.Removes) > 0 && (!claimsBroker || len(b.Emits)+len(b.State)+len(b.Invokes) > 0) { + problems = append(problems, fmt.Sprintf("%s's tools bundle %s's bus removes from the bus; only the bus "+ + "module's (the holder of mesh-broker) may, and with nothing else (novox/hq ADR 0564 Limits 4)", m.Module, a.Name)) + } + } + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "process" { + continue + } + if b, present, err := processBusOf(r); present && err == nil && len(b.Removes) > 0 { + problems = append(problems, fmt.Sprintf("%s's process %v's bus removes from the bus; only a tools bundle's "+ + "may (novox/hq ADR 0564 Limits 4)", m.Module, r["id"])) + } + } + return problems } // UnmarshalJSON reads a process's bus strictly: an unknown key — serves, a seat to hold or claim, or @@ -87,7 +191,7 @@ func (m Manifest) ProcessBuses() map[string]ProcessBus { // §3): the account rule is modelled on RunsAsProblems (seats_declared.go:522-550), and an invoked tool or // seat verb's shape on invokeProblems. func ProcessBusProblems(m Manifest) []string { - var problems []string + problems := toolsBusProblems(m) streams := map[string]bool{} for _, s := range m.Streams { streams[s.Name] = true diff --git a/go/catalogue/process_bus_test.go b/go/catalogue/process_bus_test.go index 5551ec1..bd59506 100644 --- a/go/catalogue/process_bus_test.go +++ b/go/catalogue/process_bus_test.go @@ -47,6 +47,49 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { !b.Asks || !strSliceEqual(b.Invokes, []string{"seat:operator-channel.tell"}) { t.Fatalf("the bus read as %+v", b) } + + // 1952 S5 (novox/hq ADR 0564 Limits 4; the lead, 2026-10-29): a tools bundle declares its own bus account, read as a + // process's, named after its binary and kept through resolution; a removal is from the closed set, alone, and the + // bus module's (mesh-broker) alone. + tools := func(claims, bus string) string { + return `{"module":"nats","version":"1","claims":` + claims + `,"build":{"artifacts":[{"name":"tools","kind":"bundle",` + + `"language":"go","system":"arch","from":"cmd/nats-tools","binary":"nats-tools","loads":["nats-tools"],"bus":` + bus + `}]}}` + } + broker := `[{"name":"mesh-broker","scope":"mesh"}]` + for _, c := range []struct { + name, manifest, refused string + }{ + {name: "the bus module's tools removing", manifest: tools(broker, `{"removes":["DEAD_LETTER_NOTICES.controller","mesh-host_acts"]}`)}, + {name: "a removal outside the set", manifest: tools(broker, `{"removes":["EVENTS.a_m"]}`), refused: "not one of"}, + {name: "a removal beside a grant", manifest: tools(broker, `{"removes":["mesh-host_acts"],"invokes":["nats.nats_streams"]}`), refused: "with nothing else"}, + {name: "another module's tools removing", manifest: tools(`[]`, `{"removes":["mesh-host_acts"]}`), refused: "only the bus module"}, + {name: "a tools bundle consuming a queue", manifest: tools(broker, `{"queues":["x"]}`), refused: "holds no consumer"}, + {name: "a process removing", manifest: mailSenderProcessManifest(t, `{"removes":["mesh-host_acts"]}`), refused: "only a tools bundle"}, + } { + m, err := ParseManifest([]byte(c.manifest)) + if c.refused != "" { + if err == nil || !strings.Contains(err.Error(), c.refused) { + t.Errorf("%s: %v, want refused saying %q", c.name, err, c.refused) + } + continue + } + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + accounts := m.BusAccounts() + if len(accounts) != 1 || accounts[0].ID != "nats-tools" || !accounts[0].Tools || + !strSliceEqual(accounts[0].Bus.Removes, []string{RemovesControllerDeadLetterConsumer, RemovesEngineActs}) { + t.Errorf("%s: the accounts are %+v", c.name, accounts) + } + // Resolved, the build goes and its bundles keep the bus (build.go). + resolved := Manifest{Module: "nats", Bundles: []Bundle{{Name: "tools", Binary: "nats-tools", Bus: m.Build.Artifacts[0].Bus}}} + if got := resolved.BusAccounts(); len(got) != 1 || got[0].ID != "nats-tools" { + t.Errorf("%s: resolved, the accounts are %+v", c.name, got) + } + } + if got := m.BusAccounts(); len(got) != 1 || got[0].ID != "send" || got[0].Tools { + t.Errorf("the process's account is %+v", got) + } } func strSliceEqual(a, b []string) bool { From b6ef01d6c6a7db4f45b2eaa44365132c42466993 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:57:50 +0100 Subject: [PATCH 14/52] Let the watchman hear the bus module's slow consumers and closed connections and read the provisioner reports, and never the bus's API audit, so S9 and S21 have their facts without the vault's hashes (issue 1952 S5) --- go/buscompose/buscompose_test.go | 10 +++++++++- go/buscompose/health_grants.go | 12 +++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index e4f8212..968a7c8 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -656,11 +656,19 @@ func healthChecks(t *testing.T) { if name == WatchmanModule { // It hears its watchdogs' words on core subjects (Limits 3, 11). for _, subject := range []string{"mesh.mod.power.event.sleeping", "mesh.control.g14.tools-alive", - "mesh.seat.mesh-self-check.event.checked", "mesh.mod.keycloak.event.provisioner.failing"} { + "mesh.seat.mesh-self-check.event.checked", "mesh.mod.keycloak.event.provisioner.failing", + "mesh.mod.nats.event.bus.slow-consumer", "prov.applied.g14.keycloak"} { if !MaySubscribe(perms, subject) { t.Errorf("the watchman may not hear %s", subject) } } + // Never the bus's API audit, which carries every API answer in the account (the lead, 2026-10-29). + if MaySubscribe(perms, "$JS.EVENT.ADVISORY.API") || MaySubscribe(perms, "$JS.EVENT.ADVISORY.API.x") { + t.Error("the watchman hears the bus's API audit") + } + if !MayPublish(perms, "$JS.API.DIRECT.GET.PROV_APPLIED.prov.applied.g14.keycloak") { + t.Error("the watchman may not read the newest provisioner report") + } } subjects := append(append([]string(nil), may[name]...), may[name+" sources"]...) denied := append(append([]string(nil), mayNot[name]...), mayNot[name+" sources"]...) diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go index e795a85..e31a935 100644 --- a/go/buscompose/health_grants.go +++ b/go/buscompose/health_grants.go @@ -89,6 +89,10 @@ func WatchmanDeadLetterGrants(sources []string) []string { "$JS.API.STREAM.INFO." + DeadLettersStream, "$JS.API.STREAM.MSG.GET." + DeadLettersStream, deadLetterPrefix + ">", + // And the provisioner clients' reports, the newest per subject read at its start (S21, ADR 0362 §3). + "$JS.API.STREAM.INFO." + ProvAppliedStream, + "$JS.API.STREAM.MSG.GET." + ProvAppliedStream, + "$JS.API.DIRECT.GET." + ProvAppliedStream + ".>", } for _, s := range DeadLetterSources(sources) { out = append(out, "$JS.API.STREAM.INFO."+s, "$JS.API.STREAM.MSG.GET."+s) @@ -98,7 +102,10 @@ func WatchmanDeadLetterGrants(sources []string) []string { // WatchmanHears are the core subjects the watchman subscribes, with no durable consumer (ADR 0564 clauses 4, 5 and // Limits 3, 11): the power seat's words (S1, ADR 0211), the providers' failing and recovered words (S8), each tool -// runner's beat (S11), the self-check's checked (S10), a consumer deleted (S9), and the dead letters kept. +// runner's beat (S11), the self-check's checked (S10), a consumer deleted and the bus module's slow consumers and closed +// connections (S9), the provisioner clients' reports (S21), and the dead letters kept. Never the bus's API audit +// (`$JS.EVENT.ADVISORY.API`): it carries every API call's request and answer in the account, the vault's hashes among +// them (the lead, 2026-10-29). func WatchmanHears() []string { return []string{ "mesh.mod.power.event.*", @@ -107,6 +114,9 @@ func WatchmanHears() []string { "mesh.control.*.tools-alive", "mesh.seat.mesh-self-check.event." + health.EventChecked, "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", + "mesh.mod.nats.event.bus.slow-consumer", + "mesh.mod.nats.event.bus.connection-closed", + ProvAppliedSubjects, deadLetterPrefix + ">", } } From 8e3d7a4c3c3596261975c46ac1aedf8432324f8d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:59:11 +0100 Subject: [PATCH 15/52] Give a tools bundle's bus-account secret to the runtime's account and refuse it where that account cannot reach, so the bus tools can read their credential (issue 1952 S5, ADR 0564 Limits 4) --- go/catalogue/declaration.go | 38 +++++++++++-------------- go/catalogue/process_bus.go | 48 +++++++++++++++++++++++++++++++- go/catalogue/process_bus_test.go | 17 +++++++++-- 3 files changed, 78 insertions(+), 25 deletions(-) diff --git a/go/catalogue/declaration.go b/go/catalogue/declaration.go index 6a96e6e..6389454 100644 --- a/go/catalogue/declaration.go +++ b/go/catalogue/declaration.go @@ -700,13 +700,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, handsSecrets := with.Handing(m) for _, name := range sortedKeys(m.OwnSecrets.Plain()) { if handsSecrets { - owner := m.SecretsOwner - if m.Module == RuntimeModule && r.Account != "" { - owner = r.Account - } - if o := m.OwnSecrets[name].Owner; o != "" { - owner = o - } + owner := r.ownSecretOwner(m, name) first = append(first, ownedBy(owner, map[string]any{ "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, HandedSecretField: name, })) @@ -718,13 +712,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, if string(box) == string(ForeseenVaultBox) { *foreseen = append(*foreseen, m.Module+"/"+name) } - owner := m.SecretsOwner - if m.Module == RuntimeModule && r.Account != "" { - owner = r.Account - } - if o := m.OwnSecrets[name].Owner; o != "" { - owner = o - } + owner := r.ownSecretOwner(m, name) first = append(first, ownedBy(owner, map[string]any{ "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, VaultSealedField: box, })) @@ -754,14 +742,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, // the manifest, because a manifest cannot say ${machine:account} safely — a node with no // account has nothing to resolve it to, and then the runtime runs as root and the file // stays root's. - owner := m.SecretsOwner - if m.Module == RuntimeModule && r.Account != "" { - owner = r.Account - } - if o := m.OwnSecrets[name].Owner; o != "" { - owner = o - } - first = append(first, ownedBy(owner, map[string]any{ + first = append(first, ownedBy(r.ownSecretOwner(m, name), map[string]any{ "id": NeedID(name), "type": "file", "path": m.OwnSecrets[name].Path, "sealed": sealed, })) } @@ -1677,6 +1658,19 @@ func (r Resolution) provisionsAs(m Manifest) string { return m.SecretsOwner } +// ownSecretOwner is who one of a module's own secret files belongs to: the account the manifest names for it, else the +// runtime's account for the runtime's own credential and for a tools bundle's bus account (ToolsBusSecret; novox/hq ADR +// 0564 Limits 4), which the runtime runs that bundle as, else the module's secrets-owner (root when none). +func (r Resolution) ownSecretOwner(m Manifest, name string) string { + if o := m.OwnSecrets[name].Owner; o != "" { + return o + } + if r.Account != "" && (m.Module == RuntimeModule || ToolsBusSecret(m, name)) { + return r.Account + } + return m.SecretsOwner +} + // grantPath is where one consumer's sealed credential lands on the providing machine. // // Suffixed, so the directory can also hold whatever the module writing it keeps there and so a diff --git a/go/catalogue/process_bus.go b/go/catalogue/process_bus.go index c48f11c..3477691 100644 --- a/go/catalogue/process_bus.go +++ b/go/catalogue/process_bus.go @@ -85,6 +85,52 @@ func (m Manifest) BusAccounts() []BusAccount { return out } +// ToolsBusSecret says name is the own secret a tools bundle's bus account is held in: `broker-` for a bundle +// whose bus the manifest declares (ADR 0564 Limits 4). Its file belongs to the runtime's account, which runs the bundle. +func ToolsBusSecret(m Manifest, name string) bool { + for _, a := range m.BusAccounts() { + if a.Tools && name == "broker-"+a.ID { + return true + } + } + return false +} + +// toolsSecretProblems refuses a tools bundle's bus account secret the runtime's account could not read: it is declared +// as an own secret `broker-`, at a path in a directory resource of the module that others may pass through +// (mode with its last digit odd, such as 0711 or 0755); the file itself is 0600 and the runtime account's. +func toolsSecretProblems(m Manifest) []string { + var problems []string + for _, a := range m.BusAccounts() { + if !a.Tools { + continue + } + name := "broker-" + a.ID + own, declared := m.OwnSecrets.Plain()[name] + if !declared { + problems = append(problems, fmt.Sprintf("%s's tools bundle %s declares bus and no own secret %s to hold its "+ + "account in (novox/hq ADR 0564 Limits 4)", m.Module, a.ID, name)) + continue + } + dir, _, _ := strings.Cut(strings.TrimPrefix(own.Path, "${dir:"), "}") + passable := false + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" || fmt.Sprint(r["id"]) != dir || !strings.HasPrefix(own.Path, "${dir:"+dir+"}/") { + continue + } + mode, _ := r["mode"].(string) + passable = mode != "" && strings.ContainsAny(mode[len(mode)-1:], "1357") + } + if !passable { + problems = append(problems, fmt.Sprintf("%s's own secret %s is at %s; a tools bundle's account is read by the "+ + "node's tool runtime, so it lies in a directory resource of the module that others may pass through "+ + "(mode 0711 or 0755, `${dir:}/%s`), the file itself the runtime account's, 0600 (novox/hq ADR 0564 "+ + "Limits 4)", m.Module, name, own.Path, name)) + } + } + return problems +} + // toolsBusProblems is what is wrong with a tools bundle's bus (ADR 0564 Limits 4): a bus on anything but a bundle // compiled to a binary the runtime loads, a removal outside the closed set, or a removal beside any other grant or by a // module that does not hold mesh-broker. Every other field is judged as a process's. @@ -191,7 +237,7 @@ func (m Manifest) ProcessBuses() map[string]ProcessBus { // §3): the account rule is modelled on RunsAsProblems (seats_declared.go:522-550), and an invoked tool or // seat verb's shape on invokeProblems. func ProcessBusProblems(m Manifest) []string { - problems := toolsBusProblems(m) + problems := append(toolsBusProblems(m), toolsSecretProblems(m)...) streams := map[string]bool{} for _, s := range m.Streams { streams[s.Name] = true diff --git a/go/catalogue/process_bus_test.go b/go/catalogue/process_bus_test.go index bd59506..cbd9b37 100644 --- a/go/catalogue/process_bus_test.go +++ b/go/catalogue/process_bus_test.go @@ -51,10 +51,14 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { // 1952 S5 (novox/hq ADR 0564 Limits 4; the lead, 2026-10-29): a tools bundle declares its own bus account, read as a // process's, named after its binary and kept through resolution; a removal is from the closed set, alone, and the // bus module's (mesh-broker) alone. - tools := func(claims, bus string) string { - return `{"module":"nats","version":"1","claims":` + claims + `,"build":{"artifacts":[{"name":"tools","kind":"bundle",` + + toolsIn := func(claims, bus, mode string) string { + return `{"module":"nats","version":"1","claims":` + claims + `,"own-secrets":{"broker":"${dir:mesh-state}/broker",` + + `"broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"},"resources":[{"id":"mesh-state","type":"directory",` + + `"mode":"0700","place":"mesh","unknown":"program"},{"id":"tools-secret","type":"directory","mode":"` + mode + + `","place":"mesh","unknown":"program"}],"build":{"artifacts":[{"name":"tools","kind":"bundle",` + `"language":"go","system":"arch","from":"cmd/nats-tools","binary":"nats-tools","loads":["nats-tools"],"bus":` + bus + `}]}}` } + tools := func(claims, bus string) string { return toolsIn(claims, bus, "0711") } broker := `[{"name":"mesh-broker","scope":"mesh"}]` for _, c := range []struct { name, manifest, refused string @@ -65,6 +69,9 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { {name: "another module's tools removing", manifest: tools(`[]`, `{"removes":["mesh-host_acts"]}`), refused: "only the bus module"}, {name: "a tools bundle consuming a queue", manifest: tools(broker, `{"queues":["x"]}`), refused: "holds no consumer"}, {name: "a process removing", manifest: mailSenderProcessManifest(t, `{"removes":["mesh-host_acts"]}`), refused: "only a tools bundle"}, + // The runtime's account reads the account's secret: it lies where others may pass (the host agent, 2026-10-29). + {name: "a tools secret the runtime cannot reach", manifest: toolsIn(broker, `{"removes":["mesh-host_acts"]}`, "0700"), + refused: "others may pass through"}, } { m, err := ParseManifest([]byte(c.manifest)) if c.refused != "" { @@ -81,6 +88,12 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { !strSliceEqual(accounts[0].Bus.Removes, []string{RemovesControllerDeadLetterConsumer, RemovesEngineActs}) { t.Errorf("%s: the accounts are %+v", c.name, accounts) } + // Its secret's file is the runtime account's, as the runtime's own credential is; the module's other secrets stay + // its own. + if (Resolution{Account: "ops"}).ownSecretOwner(m, "broker-nats-tools") != "ops" || + (Resolution{Account: "ops"}).ownSecretOwner(m, "broker") != "" || (Resolution{}).ownSecretOwner(m, "broker-nats-tools") != "" { + t.Errorf("%s: the tools secret's owner is %q", c.name, (Resolution{Account: "ops"}).ownSecretOwner(m, "broker-nats-tools")) + } // Resolved, the build goes and its bundles keep the bus (build.go). resolved := Manifest{Module: "nats", Bundles: []Bundle{{Name: "tools", Binary: "nats-tools", Bus: m.Build.Artifacts[0].Bus}}} if got := resolved.BusAccounts(); len(got) != 1 || got[0].ID != "nats-tools" { From 32f5ce3712687ee3fbf8d5bffb2f268f8bbdc294 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:00:43 +0100 Subject: [PATCH 16/52] Refuse a tools account secret under a parent the runtime cannot pass, so the rule covers the module's 0700 mesh place too (issue 1952 S5, ADR 0564 Limits 4) --- go/catalogue/process_bus.go | 63 ++++++++++++++++++++++++-------- go/catalogue/process_bus_test.go | 13 ++++--- 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/go/catalogue/process_bus.go b/go/catalogue/process_bus.go index 3477691..25dea20 100644 --- a/go/catalogue/process_bus.go +++ b/go/catalogue/process_bus.go @@ -97,8 +97,8 @@ func ToolsBusSecret(m Manifest, name string) bool { } // toolsSecretProblems refuses a tools bundle's bus account secret the runtime's account could not read: it is declared -// as an own secret `broker-`, at a path in a directory resource of the module that others may pass through -// (mode with its last digit odd, such as 0711 or 0755); the file itself is 0600 and the runtime account's. +// as an own secret `broker-`, at `${dir:}/broker-` for a directory resource every parent of which the +// runtime's account may pass (passableDir); the file itself is 0600 and the runtime account's. func toolsSecretProblems(m Manifest) []string { var problems []string for _, a := range m.BusAccounts() { @@ -113,24 +113,57 @@ func toolsSecretProblems(m Manifest) []string { continue } dir, _, _ := strings.Cut(strings.TrimPrefix(own.Path, "${dir:"), "}") - passable := false - for _, r := range m.Resources { - if fmt.Sprint(r["type"]) != "directory" || fmt.Sprint(r["id"]) != dir || !strings.HasPrefix(own.Path, "${dir:"+dir+"}/") { - continue - } - mode, _ := r["mode"].(string) - passable = mode != "" && strings.ContainsAny(mode[len(mode)-1:], "1357") - } - if !passable { - problems = append(problems, fmt.Sprintf("%s's own secret %s is at %s; a tools bundle's account is read by the "+ - "node's tool runtime, so it lies in a directory resource of the module that others may pass through "+ - "(mode 0711 or 0755, `${dir:}/%s`), the file itself the runtime account's, 0600 (novox/hq ADR 0564 "+ - "Limits 4)", m.Module, name, own.Path, name)) + if !strings.HasPrefix(own.Path, "${dir:"+dir+"}/") || !passableDir(m, dir, 0) { + problems = append(problems, fmt.Sprintf("%s's own secret %s is at %s, which the node's tool runtime cannot "+ + "reach: a tools bundle's account lies in a directory resource the runtime's account may pass, every parent "+ + "included, such as {\"id\":\"tools-secret\",\"type\":\"directory\",\"path\":\"%s/%s-tools\",\"mode\":\"0711\"} "+ + "with the secret at ${dir:tools-secret}/%s; never under the module's own or mesh place, which are its "+ + "own (novox/hq ADR 0564 Limits 4)", m.Module, name, own.Path, meshStateRoot, m.Module, name)) } } return problems } +// meshStateRoot is the mesh's directory of every module's mesh place on a node of the default data root: root's, mode +// 0755, so the runtime's account passes it, while each module's own place beneath it is 0700. +const meshStateRoot = defaultDataRoot + "/" + placeMesh + +// passableDir says the runtime's account may pass the directory resource id and every parent the mesh makes for it: its +// own mode lets others pass (the last digit odd), and it is either stated directly under meshStateRoot or beneath a +// placed directory that is passable in turn. A pathless directory resolves under the module's own root or its mesh +// place, which are the module's own, so it never is. +func passableDir(m Manifest, id string, depth int) bool { + if depth > 4 { + return false + } + for _, r := range m.Resources { + if fmt.Sprint(r["type"]) != "directory" || fmt.Sprint(r["id"]) != id { + continue + } + mode, _ := r["mode"].(string) + if mode == "" || !strings.ContainsAny(mode[len(mode)-1:], "1357") { + return false + } + path, _ := r["path"].(string) + switch { + case strings.HasPrefix(path, "${dir:"): + parent, _, _ := strings.Cut(strings.TrimPrefix(path, "${dir:"), "}") + return passableDir(m, parent, depth+1) + case strings.HasPrefix(path, "/"): + return filepathDir(strings.TrimRight(path, "/")) == meshStateRoot + } + return false + } + return false +} + +func filepathDir(p string) string { + if i := strings.LastIndex(p, "/"); i > 0 { + return p[:i] + } + return "/" +} + // toolsBusProblems is what is wrong with a tools bundle's bus (ADR 0564 Limits 4): a bus on anything but a bundle // compiled to a binary the runtime loads, a removal outside the closed set, or a removal beside any other grant or by a // module that does not hold mesh-broker. Every other field is judged as a process's. diff --git a/go/catalogue/process_bus_test.go b/go/catalogue/process_bus_test.go index cbd9b37..7b3f8f3 100644 --- a/go/catalogue/process_bus_test.go +++ b/go/catalogue/process_bus_test.go @@ -51,14 +51,14 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { // 1952 S5 (novox/hq ADR 0564 Limits 4; the lead, 2026-10-29): a tools bundle declares its own bus account, read as a // process's, named after its binary and kept through resolution; a removal is from the closed set, alone, and the // bus module's (mesh-broker) alone. - toolsIn := func(claims, bus, mode string) string { + toolsIn := func(claims, bus, mode, where string) string { return `{"module":"nats","version":"1","claims":` + claims + `,"own-secrets":{"broker":"${dir:mesh-state}/broker",` + `"broker-nats-tools":"${dir:tools-secret}/broker-nats-tools"},"resources":[{"id":"mesh-state","type":"directory",` + `"mode":"0700","place":"mesh","unknown":"program"},{"id":"tools-secret","type":"directory","mode":"` + mode + - `","place":"mesh","unknown":"program"}],"build":{"artifacts":[{"name":"tools","kind":"bundle",` + + `","path":"` + where + `","unknown":"program"}],"build":{"artifacts":[{"name":"tools","kind":"bundle",` + `"language":"go","system":"arch","from":"cmd/nats-tools","binary":"nats-tools","loads":["nats-tools"],"bus":` + bus + `}]}}` } - tools := func(claims, bus string) string { return toolsIn(claims, bus, "0711") } + tools := func(claims, bus string) string { return toolsIn(claims, bus, "0711", "/var/lib/mesh/nats-tools") } broker := `[{"name":"mesh-broker","scope":"mesh"}]` for _, c := range []struct { name, manifest, refused string @@ -70,8 +70,11 @@ func TestAProcessDeclaresItsBusAndIsAccepted(t *testing.T) { {name: "a tools bundle consuming a queue", manifest: tools(broker, `{"queues":["x"]}`), refused: "holds no consumer"}, {name: "a process removing", manifest: mailSenderProcessManifest(t, `{"removes":["mesh-host_acts"]}`), refused: "only a tools bundle"}, // The runtime's account reads the account's secret: it lies where others may pass (the host agent, 2026-10-29). - {name: "a tools secret the runtime cannot reach", manifest: toolsIn(broker, `{"removes":["mesh-host_acts"]}`, "0700"), - refused: "others may pass through"}, + {name: "a tools secret the runtime cannot reach", manifest: toolsIn(broker, `{"removes":["mesh-host_acts"]}`, "0700", + "/var/lib/mesh/nats-tools"), refused: "cannot reach"}, + // Its parents too: under the module's mesh place, /var/lib/mesh/nats is 0700 (the lead, read on novox 2026-10-29). + {name: "a tools secret beneath the module's mesh place", manifest: toolsIn(broker, `{"removes":["mesh-host_acts"]}`, "0711", + "${dir:mesh-state}/tools"), refused: "cannot reach"}, } { m, err := ParseManifest([]byte(c.manifest)) if c.refused != "" { From 7df90445ad138cfdbea7fb625c782a4232abcaa9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:09:16 +0100 Subject: [PATCH 17/52] Add the seats mesh-wants and mesh-hand-acts, the engine's packages verb, the act layout and a recorder that falls back to the controller's log while the seat has no holder, so the control records have homes the lease does not hold (issue 1953 L5-S1, S2, S4) --- go/buscompose/buscompose_test.go | 8 +- go/buscompose/userlist_switch.go | 7 +- go/catalogue/control_records_seats.go | 64 +++++++++++ go/catalogue/engine_seat.go | 6 + go/catalogue/seats.go | 4 + go/catalogue/seats_test.go | 18 +-- go/handacts/handacts.go | 158 ++++++++++++++++++++++++++ go/handacts/handacts_test.go | 74 ++++++++++++ 8 files changed, 328 insertions(+), 11 deletions(-) create mode 100644 go/catalogue/control_records_seats.go create mode 100644 go/handacts/handacts.go create mode 100644 go/handacts/handacts_test.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 968a7c8..004a152 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -581,6 +581,8 @@ func healthChecks(t *testing.T) { {"mesh-self-check", ControllerLease, false}, {"messenger", ControllerLease, false}, {"mesh-conditions", ControllerConditions, true}, {"mesh-healers", ControllerConditions, false}, {"messenger", ControllerConditions, false}, {"mesh-healers", ControllerHandActs, true}, {"mesh-conditions", ControllerHandActs, false}, + // ADR 0565 clauses 10, 11 (issue 1953 L5-S4): mesh-hand-acts reads the controller's log beside its own. + {"mesh-hand-acts", ControllerHandActs, true}, {"mesh-conditions", "mesh-healers.attempts", true}, {"mesh-conditions", HealersWould, false}, {"mesh-delivery-view", HealersWould, false}, } { granted, _ := ReadsAnnouncedBy(c.reader, []string{c.read}, announced) @@ -595,6 +597,7 @@ func healthChecks(t *testing.T) { {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, + {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools", @@ -606,6 +609,7 @@ func healthChecks(t *testing.T) { } may := map[string][]string{ "a.mesh-conditions": {"$KV.mesh-conditions_conditions.machine.g14.silent"}, + "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.INFO.SEAT_TELEGRAM_SENDER", @@ -617,7 +621,9 @@ func healthChecks(t *testing.T) { // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, - "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, + // Nor mesh-hand-acts' log and warrants, which it alone writes (ADR 0565 clause 10). + "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", + "$KV.mesh-hand-acts_warrants.x", "mesh.log.mesh-hand-acts.acts.x"}, // The watchman never deletes or changes the controller's consumer (clause 5). // Its reads and deletes of a message are on the streams a dead letter comes from, named, never a bucket's // (the lead, 2026-10-29). diff --git a/go/buscompose/userlist_switch.go b/go/buscompose/userlist_switch.go index 266eaad..bae9341 100644 --- a/go/buscompose/userlist_switch.go +++ b/go/buscompose/userlist_switch.go @@ -5,6 +5,7 @@ import ( "strings" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -84,11 +85,11 @@ const VaultBusAccounts = "mesh-vault.bus-accounts" // reads the catalogue's placement alone from then (ADR 0549 §5, issue 774). // // The health holders (novox/hq ADR 0564 Limits 2 and 8, clauses 8 and 9): the condition keeper, mesh-healers and the -// watchman read the lease; the keeper alone the controller's conditions, mesh-healers alone its hand-act log; and nobody -// the healers' `would` (health_grants.go). +// watchman read the lease; the keeper alone the controller's conditions; mesh-healers and mesh-hand-acts its hand-act log +// (ADR 0565 clauses 10, 11: every reader of acts reads both logs); and nobody the healers' `would` (health_grants.go). var NamedReaders = map[string][]string{VaultBusAccounts: {GrantsModule}, GrantsAccounts: {VaultSeat}, ControllerLease: {VaultSeat, GrantsModule, delivery.AssignerModule, health.KeeperModule, health.HealersModule, WatchmanModule}, - ControllerConditions: {health.KeeperModule}, ControllerHandActs: {health.HealersModule}, HealersWould: {}} + ControllerConditions: {health.KeeperModule}, ControllerHandActs: {health.HealersModule, handacts.Module}, HealersWould: {}} // ReaderMay says whether module may read the state read (`.`) by the named readers: always, for a state // that names none. diff --git a/go/catalogue/control_records_seats.go b/go/catalogue/control_records_seats.go new file mode 100644 index 0000000..5d30fd0 --- /dev/null +++ b/go/catalogue/control_records_seats.go @@ -0,0 +1,64 @@ +package catalogue + +// **The control records, out of the controller** (novox/hq ADR 0565 clause 1, issue 1953): two seats at mesh scope, +// each held once, and the node seat's verb `packages`. +// +// - mesh-wants, held by the assigner (mesh-delivery-assigner): a person's upgrade policy and the bus step (clauses 7, +// 8, 9); +// - mesh-hand-acts, held by the module mesh-hand-acts: the hand-act log and the warrants recorded (clause 10). +// +// Their events are their holders' to say on the seat; their layouts are go/handacts' and go/delivery's. + +// The two seats. +const ( + WantsSeat = "mesh-wants" + HandActsSeat = "mesh-hand-acts" +) + +// controlRecordsDecision is the record that made the two seats. +const controlRecordsDecision = "novox/hq ADR 0565" + +// wantsVerbs are the assigner's verbs on mesh-wants (ADR 0565 clauses 7, 8, 9, 15). +func wantsVerbs() []Verb { + return []Verb{ + {Name: "upgrade", Description: "Whether each module's new builds wait for a person: its policy (roll, together or record) " + + "and where it comes from (a person, the module, the bus, its irreplaceable data, the default); the bus is never " + + "rolled. With module, that one; with module, policy and why, a person's choice for it, recorded as a hand act; " + + "with policy default, the choice taken back. With compare, every module whose policy differs from the one the " + + "controller writes beside its compositions.", + Input: schema(map[string]string{ + "module": "one module", + "policy": "with module: roll, together, record, or default to take a person's choice back", + "why": "with policy: why, required and kept", + "compare": "\"true\": every module whose policy at the assigner differs from the controller's", + }, nil, "compare")}, + {Name: "bus", Description: "What a bus upgrade would do: the fingerprint of the bus module each node runs against the " + + "indexer's newest, and the last bus steps from the hand-act log and the assigner's puts of the bus. Read-only: " + + "the step itself is the put on the operator's warrant.", + Input: schema(map[string]string{}, nil)}, + } +} + +// handActsVerbs are mesh-hand-acts' verbs (ADR 0565 clause 10), as the controller's took them. +func handActsVerbs() []Verb { + return []Verb{ + {Name: "hand-act", Description: "Record one act done by hand: what, why, its cause in a word or a condition's kind " + + "(the word a second act for the same reason uses) and the condition it addresses. Who did it is the caller the " + + "runtime proves, never said by the caller. Said as act-recorded.", + Input: schema(map[string]string{ + "what": "what was done, in a line", + "why": "why it had to be done by hand", + "cause": "the cause in a word, or a condition's kind", + "condition": "the key of the condition it addressed, if any (optional)", + "verb": "the act's verb in the log: a repair when absent, `hand-act drill` for a drill, or a recorder's own", + }, []string{"what", "why"})}, + {Name: "hand-acts", Description: "What was done by hand lately, over this log and the controller's: who, why and the " + + "cause of each, and the causes recorded twice within a fortnight (a healer is wanted).", + Input: schema(map[string]string{"days": "how many days back (default 14)", "cause": "only this cause", + "who": "only acts whose who contains this", "warrants": "\"true\": every warrant recorded"}, nil, "warrants")}, + {Name: "warranted", Description: "Record what a module did on the operator's warrant: who chose, through which channel, " + + "with which proofs and which answer read from the router's own record of that module's ask, never from the " + + "caller; once per ask. Refused for an ask still open, ended without a choice, or not that module's.", + Input: schema(map[string]string{"asker": "the module that asked", "ask": "its ask's id"}, []string{"asker", "ask"})}, + } +} diff --git a/go/catalogue/engine_seat.go b/go/catalogue/engine_seat.go index b3b695b..b924506 100644 --- a/go/catalogue/engine_seat.go +++ b/go/catalogue/engine_seat.go @@ -43,6 +43,12 @@ func EngineVerbs() []Verb { "each hand-over, access act and setuid search, open, done or refused, with who asked, the ask and the " + "operator's warrant. Read-only.", Input: schema(map[string]string{"limit": "at most this many, newest", "id": "one act, by its id"}, nil)}, + // The packages this node's compositions declare (novox/hq ADR 0565 §6, §15): optional until every engine serves it. + {Name: "packages", Optional: true, Description: "The packages this machine's compositions declare, present or absent, " + + "each with the modules that declare it, as the engine keeps them in its key of `packages`; until the controller " + + "hands its lease away, beside the controller's mirror for this machine and their difference. With write, the " + + "key written again from what the engine composed.", + Input: schema(map[string]string{"write": "\"true\": write this machine's key again from what the engine composed"}, nil, "write")}, } } diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 3082ce8..1b70e38 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -287,6 +287,10 @@ var defaultSeats = append([]Seat{ {Name: WatchdogsSeat, Scope: ScopeMesh, Decision: healthDecision, Serves: watchdogsVerbs()}, {Name: SelfCheckSeat, Scope: ScopeMesh, Decision: healthDecision, Emits: []string{"checked"}, Serves: selfCheckVerbs()}, {Name: HealersSeat, Scope: ScopeMesh, Decision: healthDecision, Emits: []string{"healer-acted"}, Serves: healersVerbs()}, + // The control records, out of the controller (novox/hq ADR 0565 clause 1, control_records_seats.go): a person's + // policy and the bus step at the assigner, and the hand-act log. + {Name: WantsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"policy-changed"}, Serves: wantsVerbs()}, + {Name: HandActsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"act-recorded"}, Serves: handActsVerbs()}, // Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a // delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight. // They keep their names until that migration is done deliberately, apart from the node-* pass. diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index bd22358..168056e 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -71,9 +71,12 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { {WatchdogsSeat, "signals,stuck,presence,dead-letters,resync", ""}, {SelfCheckSeat, "doctor,probes", "checked"}, {HealersSeat, "healers,heal,drill", "healer-acted"}, + // And ADR 0565 clause 1's two. + {WantsSeat, "upgrade,bus", "policy-changed"}, + {HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"}, } { s, ok := SeatNamed(c.seat) - if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || s.Decision != "novox/hq ADR 0564" || + if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.HasPrefix(s.Decision, "novox/hq ADR 056") || strings.Join(VerbNames(s.Serves), ",") != c.verbs || strings.Join(s.Emits, ",") != c.emits { t.Errorf("the health seat %s is %v, %+v", c.seat, ok, s) } @@ -88,10 +91,10 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { verbs = append(verbs, v.Name) } // acts reads one act by its id too (ADR 0563 §9: the seat's tools are its command line). - if props, _ := engine.Serves[len(engine.Serves)-1].Input["properties"].(map[string]any); props["id"] == nil { - t.Errorf("acts takes no id: %v", engine.Serves[len(engine.Serves)-1].Input) + if props, _ := engine.Serves[5].Input["properties"].(map[string]any); props["id"] == nil { + t.Errorf("acts takes no id: %v", engine.Serves[5].Input) } - if strings.Join(verbs, ",") != "hand-over,access-act,judge-directories,setuid-search,root-free,acts" || + if strings.Join(verbs, ",") != "hand-over,access-act,judge-directories,setuid-search,root-free,acts,packages" || strings.Join(engine.Emits, ",") != "asked,done,refused" { t.Errorf("node-engine serves %v and emits %v", verbs, engine.Emits) } @@ -114,9 +117,10 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired // mesh-build-machine row goes, when no registered manifest claims it. // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1); - // fifty-three with the four health seats (novox/hq ADR 0564 clause 1). - if len(Seats()) != 53 { - t.Errorf("the mesh defines %d seats rather than 53; the set is closed, so a change here is "+ + // fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts + // (novox/hq ADR 0565 clause 1). + if len(Seats()) != 55 { + t.Errorf("the mesh defines %d seats rather than 55; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames()) } } diff --git a/go/handacts/handacts.go b/go/handacts/handacts.go new file mode 100644 index 0000000..4ce5937 --- /dev/null +++ b/go/handacts/handacts.go @@ -0,0 +1,158 @@ +// Package handacts is the hand-act log's contract (novox/hq ADR 0565 clauses 10 and 11, issue 1953): the layout of an act, +// moved unchanged from the controller's (mesh-controller internal/link HandAct) so one reader reads both logs alike; the +// names of the seat mesh-hand-acts and of what its holder keeps; and the recorder every module records through, which +// asks the seat and, while no holder answers, the controller's verb until the controller hands its lease away, saying +// which log took the act. +package handacts + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// The seat, its holder and what it keeps (ADR 0565 clause 10). +const ( + Seat = "mesh-hand-acts" + Module = "mesh-hand-acts" + // ActsLog is the log of every act, kept 400 days; WarrantsState one key `.` per warrant, made once. + ActsLog = "acts" + WarrantsState = "warrants" + // EventRecorded is said on the seat for every act, with the act as its body. + EventRecorded = "act-recorded" + // ControllerActs is the controller's log as a reader names it, read until its lease ends. + ControllerActs = "mesh-controller.hand-acts" +) + +// KeptFor is how long an act is kept. +const KeptFor = 400 * 24 * time.Hour + +// The verbs an act is recorded under, as the controller's log names them. +const ( + VerbRecord = "hand-act record" + VerbDrill = "hand-act drill" + VerbWarrant = "warrant" + // CauseDrill is a drill's cause, never counted as a repair (S15); CauseOperatorAnswer an act the operator chose. + CauseDrill = "drill" + CauseOperatorAnswer = "operator-answer" + // KindRecordedBuilds is a push that only moved recorded builds: no repair (ADR 0242). + KindRecordedBuilds = "recorded-builds" +) + +// Act is one entry of the log, the controller's HandAct unchanged. +type Act struct { + ID string `json:"id"` + At time.Time `json:"at"` + // By is who: the caller the runtime proved. + By string `json:"by"` + // Verb and Args are the act as given. + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` + Why string `json:"why"` + // Cause is the condition kind, or a word the person gives; the verb's own name when neither. + Cause string `json:"cause"` + Condition string `json:"condition,omitempty"` + // Kind is what the act was read to be from what it did, never a word a person gives. + Kind string `json:"kind,omitempty"` + Carried []string `json:"carried,omitempty"` + // Via, Ask, Proofs and RequestedBy are an act the operator chose on a warrant. + Via string `json:"via,omitempty"` + Ask string `json:"ask,omitempty"` + Proofs []string `json:"proofs,omitempty"` + RequestedBy string `json:"requested-by,omitempty"` + // Outcome is what came of an act recorded after it was done. + Outcome string `json:"outcome,omitempty"` + // Log is which log holds it, set when it is read (LogSeat or LogController); never written. + Log string `json:"log,omitempty"` +} + +// The logs an act may be held in. +const ( + LogSeat = Seat + LogController = "mesh-controller" +) + +// Repair says the act counts toward a cause repeated (S15): not a drill, not an operator's answer, not recorded builds. +func (a Act) Repair() bool { + return a.Cause != CauseDrill && a.Verb != VerbDrill && a.Cause != CauseOperatorAnswer && a.Kind != KindRecordedBuilds +} + +// Request is one act a recorder records: what, why, its cause, the condition it addresses, and its verb in the log (a +// repair when empty). +type Request struct { + What string `json:"what"` + Why string `json:"why"` + Cause string `json:"cause,omitempty"` + Condition string `json:"condition,omitempty"` + Verb string `json:"verb,omitempty"` +} + +// Taken is which log took an act, and what its holder answered. +type Taken struct { + Log string `json:"log"` + Answer json.RawMessage `json:"answer,omitempty"` +} + +// Recorder records acts for a module (ADR 0565 clause 11): through the seat mesh-hand-acts, and while the seat has no +// holder that answers, through the controller's verb until the controller hands its lease away (the release's order +// gives no seat's holder before its callers; 00-plan.md, "Missing" 1). +type Recorder struct { + // Ask calls a seat verb by its address (`seat:.`): stdio.Ask through the runtime. + Ask func(key string, body any) (json.RawMessage, error) + // LeaseAway says the controller's lease is handed away: then nothing falls back to it. Nil is never handed away. + LeaseAway func() bool +} + +// NoResponder says err is the bus finding nobody to answer: the seat has no holder now. +func NoResponder(err error) bool { + return err != nil && strings.Contains(err.Error(), "no responders") +} + +// Record records one act. +func (r Recorder) Record(q Request) (Taken, error) { + if strings.TrimSpace(q.What) == "" || strings.TrimSpace(q.Why) == "" { + return Taken{}, errors.New("a hand act says what was done and why: nothing was recorded") + } + body := map[string]any{"what": q.What, "why": q.Why} + for k, v := range map[string]string{"cause": q.Cause, "condition": q.Condition, "verb": q.Verb} { + if v != "" { + body[k] = v + } + } + controller := map[string]any{"what": q.What, "why": q.Why, "cause": q.Cause} + if q.Condition != "" { + controller["condition"] = q.Condition + } + if q.Cause == "" { + controller["cause"] = q.Verb + } + return r.either("hand-act", body, "hand-act", controller) +} + +// RecordWarrant records what the asking module did on the operator's warrant for its ask. +func (r Recorder) RecordWarrant(asker, ask string) (Taken, error) { + body := map[string]any{"asker": asker, "ask": ask} + return r.either("warranted", body, "warranted", body) +} + +func (r Recorder) either(verb string, body any, controllerVerb string, controllerBody any) (Taken, error) { + answer, err := r.Ask("seat:"+Seat+"."+verb, body) + if err == nil { + return Taken{Log: LogSeat, Answer: answer}, nil + } + if !NoResponder(err) { + return Taken{}, fmt.Errorf("%s.%s refused: %w", Seat, verb, err) + } + if r.LeaseAway != nil && r.LeaseAway() { + return Taken{}, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing "+ + "was recorded (%w)", Seat, err) + } + answer, cerr := r.Ask("seat:mesh-controller."+controllerVerb, controllerBody) + if cerr != nil { + return Taken{}, fmt.Errorf("%s has no holder that answers (%v), and the controller's %s refused too: %w", Seat, err, + controllerVerb, cerr) + } + return Taken{Log: LogController, Answer: answer}, nil +} diff --git a/go/handacts/handacts_test.go b/go/handacts/handacts_test.go new file mode 100644 index 0000000..fe56282 --- /dev/null +++ b/go/handacts/handacts_test.go @@ -0,0 +1,74 @@ +package handacts + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" +) + +// ADR 0565 clauses 10 and 11 (issue 1953 L5-S2, S4): an act reads the controller's entries unchanged, and a recorder +// records through the seat, through the controller's verb only while the seat has no holder and the lease is the +// controller's, saying which log took it, and refuses to record nowhere. +func TestAnActIsRecordedInTheLogThatAnswers(t *testing.T) { + // The shape of mesh-controller internal/link HandAct, as its bucket keeps an entry (handacts.go at 0fbb5ef3). + raw := `{"id":"act-1","at":"2026-10-28T09:00:00Z","by":"person.jochen","verb":"hand-act record","why":"a stuck consumer",` + + `"cause":"consumer-lost","condition":"bus.EVENTS.x.consumer-lost","via":"telegram","ask":"a-1","proofs":["P1"],` + + `"requested-by":"bus.EVENTS.x.consumer-lost","outcome":"done"}` + var a Act + if err := json.Unmarshal([]byte(raw), &a); err != nil { + t.Fatal(err) + } + again, _ := json.Marshal(a) + var want, got map[string]any + _ = json.Unmarshal([]byte(raw), &want) + _ = json.Unmarshal(again, &got) + if !reflect.DeepEqual(want, got) || !a.Repair() || (Act{Cause: CauseDrill}).Repair() || (Act{Verb: VerbDrill}).Repair() { + t.Errorf("the controller's entry reads back as %s", again) + } + noResponders := errors.New("nats: no responders available for request") + for _, c := range []struct { + name string + seat error + ctrl error + away bool + log string + refused string + asked string + warrants bool + }{ + {name: "the seat answers", log: LogSeat, asked: "seat:mesh-hand-acts.hand-act"}, + {name: "no holder yet", seat: noResponders, log: LogController, asked: "seat:mesh-hand-acts.hand-act,seat:mesh-controller.hand-act"}, + {name: "no holder, the lease handed away", seat: noResponders, away: true, refused: "handed its lease away", + asked: "seat:mesh-hand-acts.hand-act"}, + {name: "the seat refuses", seat: errors.New("why is required"), refused: "refused", asked: "seat:mesh-hand-acts.hand-act"}, + {name: "neither answers", seat: noResponders, ctrl: noResponders, refused: "refused too", + asked: "seat:mesh-hand-acts.hand-act,seat:mesh-controller.hand-act"}, + {name: "a warrant without a holder", seat: noResponders, warrants: true, log: LogController, + asked: "seat:mesh-hand-acts.warranted,seat:mesh-controller.warranted"}, + } { + var asked []string + r := Recorder{LeaseAway: func() bool { return c.away }, Ask: func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key) + if strings.HasPrefix(key, "seat:mesh-hand-acts.") { + return json.RawMessage(`{}`), c.seat + } + return json.RawMessage(`{}`), c.ctrl + }} + var taken Taken + var err error + if c.warrants { + taken, err = r.RecordWarrant("mesh-config", "ask-1") + } else { + taken, err = r.Record(Request{What: "silenced", Why: "known", Cause: "silent", Verb: "conditions silence"}) + } + if (c.refused != "") != (err != nil) || (err != nil && !strings.Contains(err.Error(), c.refused)) || taken.Log != c.log || + strings.Join(asked, ",") != c.asked { + t.Errorf("%s: took %q, %v, asked %v", c.name, taken.Log, err, asked) + } + } + if _, err := (Recorder{}).Record(Request{What: "x"}); err == nil { + t.Error("an act with no why was recorded") + } +} From fbd00c89436c53a36fd67cb8bb7dbaddf0ee7579 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:10:44 +0100 Subject: [PATCH 18/52] Refuse a manifest that lists its claimed seat's event among its own emits, so a holder never says a seat event where no consumer hears it (issue 1952, second of a kind) --- go/catalogue/delivery_seat.go | 25 +++++++++++++++++++++++++ go/catalogue/manifest.go | 1 + go/catalogue/seats_test.go | 12 ++++++++++++ 3 files changed, 38 insertions(+) diff --git a/go/catalogue/delivery_seat.go b/go/catalogue/delivery_seat.go index f1d2d24..5cab2ce 100644 --- a/go/catalogue/delivery_seat.go +++ b/go/catalogue/delivery_seat.go @@ -1,5 +1,7 @@ package catalogue +import "fmt" + // The delivery's owner (novox/hq ADR 0239, to-be 47): one holder for the mesh, the module mesh-delivery, // which owns a delivery — one commit in one repository, from its pull request's head to every machine — // and a delivery group, and asks the controller for every act. A seat of the mesh's own, so the role has @@ -105,6 +107,29 @@ func SeatSays(emits []string, event string) bool { return false } +// seatEventProblems refuses a module that lists among its own `emits` an event a seat it claims says (the lead, +// 2026-10-29, issue 1952, the second of a kind: the self-check's `checked` and mesh-hand-acts' `act-recorded`). A +// consumer of `.` hears the seat's subject (SeatSays), so the holder says it there, with stdio.SeatPublish +// on `mesh.seat..event.`; said through stdio.Emit as the module's own, it lands where nobody listens. Caught +// here, when the manifest is read at the build, not when nothing is heard on the machines. +func seatEventProblems(m Manifest) []string { + var problems []string + for _, c := range m.Claims { + seat, known := SeatNamed(c.Name) + if !known || len(seat.Emits) == 0 { + continue + } + for _, e := range m.Emits { + if SeatSays(seat.Emits, e) { + problems = append(problems, fmt.Sprintf("%s lists %q among its own emits, and the seat %s it claims says it: "+ + "say it on the seat with stdio.SeatPublish(\"mesh.seat.%s.event.%s\", ...) and leave it out of emits, or "+ + "no consumer of %s.%s hears it", m.Module, e, seat.Name, seat.Name, e, seat.Name, e)) + } + } + } + return problems +} + // topicMatches is whether an event's name matches a declared pattern, `*` one dot-separated segment and // `>` or `**` the rest. func topicMatches(pattern, event string) bool { diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index ca3eba9..1423d11 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -2093,6 +2093,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { problems = append(problems, declaredSeatProblems(m)...) problems = append(problems, RunsAsProblems(m)...) problems = append(problems, ProcessBusProblems(m)...) + problems = append(problems, seatEventProblems(m)...) problems = append(problems, ReadsHolderProblems(m)...) problems = append(problems, FilesHolderProblems(m)...) if m.Computed != "" && len(m.Resources) > 0 { diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index 168056e..0226c15 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -184,6 +184,18 @@ func TestAClaimOnASeatAnotherModuleDeclaresIsAccepted(t *testing.T) { if problems := CatalogueProblems(Shelf{claimant.Module: claimant, "someone": declarer}); len(problems) != 0 { t.Fatalf("a claim on a declared seat was refused: %v", problems) } + // The second of a kind (issue 1952; the lead, 2026-10-29): a holder that lists its seat's event among its own emits + // would say it where no consumer hears it, so the manifest is refused, naming the seat's subject; the seat's + // holder saying nothing of its own is accepted. + holder := func(emits string) []byte { + return []byte(`{"module":"mesh-self-check","version":"1","emits":` + emits + `,"claims":[{"name":"mesh-self-check","scope":"mesh","serves":["doctor","probes"]}]}`) + } + if _, err := ParseManifest(holder(`["checked"]`)); err == nil || !strings.Contains(err.Error(), "mesh.seat.mesh-self-check.event.checked") { + t.Errorf("a seat's event listed as the module's own: %v", err) + } + if _, err := ParseManifest(holder(`[]`)); err != nil { + t.Errorf("a holder saying its seat's event on the seat was refused: %v", err) + } } // A module may define its own seat and claim it — the mesh enforces exclusivity without knowing From 20cac82ab767cb849bbac06f5994ac1ae5494f03 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:14:01 +0100 Subject: [PATCH 19/52] Give the control records their layouts, a person's policy and its one derivation, and the engine's own packages key, so the assigner, the indexer and the nodes keep them without the controller (issue 1953 L5-S2, S3, S4) --- go/buscompose/buscompose_test.go | 18 +++++- go/buscompose/engine.go | 15 +++++ go/buscompose/writers.go | 4 ++ go/catalogue/upgrade.go | 44 +++++++++++++ go/catalogue/upgrade_test.go | 20 ++++++ go/delivery/control_records.go | 95 +++++++++++++++++++++++++++++ go/delivery/control_records_test.go | 45 ++++++++++++++ 7 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 go/delivery/control_records.go create mode 100644 go/delivery/control_records_test.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 004a152..56d0879 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -75,7 +75,9 @@ func engineBeyond(user, g string) bool { } return strings.HasPrefix(g, "mesh.seat."+EngineSeat+".") && strings.Contains(g, "."+node) || strings.HasPrefix(g, "$SRV.") || strings.Contains(g, ".node-engine-"+node) || - strings.Contains(g, "KV_"+EngineActsBucket) || strings.HasPrefix(g, "$KV."+EngineActsBucket+"."+node+".") + strings.Contains(g, "KV_"+EngineActsBucket) || strings.HasPrefix(g, "$KV."+EngineActsBucket+"."+node+".") || + // And its node's declared packages (novox/hq ADR 0565 clause 6). + strings.Contains(g, "KV_"+EnginePackagesBucket) || g == "$KV."+EnginePackagesBucket+"."+node } // createsAnAccountsConsumer is the grants part's create and read of a durable consumer an account needs, by its stream @@ -643,6 +645,20 @@ func healthChecks(t *testing.T) { "a.nats": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, "a.shop.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-conditions_conditions.x"}, } + // ADR 0565 clauses 6, 7 (issue 1953 L5-S4): a node's engine writes its own packages key and no other node's; only the + // assigner writes its policies. + for _, u := range users { + perms, err := PermissionsFor(u) + if err != nil { + continue + } + if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_packages.a") || MayPublish(perms, "$KV.mesh-host_packages.b")) { + t.Errorf("node %s writes the packages keys %v", u.Node, perms.Publish) + } + if u.Kind == KindModule && MayPublish(perms, "$KV.mesh-delivery-assigner_policies.x") { + t.Errorf("%s writes the assigner's policies", u.Username()) + } + } // The tools account's secret is the module's own broker-, which its manifest's own-secrets names and the // vault keeps by that name (AccountSecret). if w := (WantedAccount{User: "a.nats.nats-tools", Kind: KindProcess, Node: "a", Module: "nats", Process: "nats-tools", diff --git a/go/buscompose/engine.go b/go/buscompose/engine.go index 501084b..2e6a902 100644 --- a/go/buscompose/engine.go +++ b/go/buscompose/engine.go @@ -3,6 +3,7 @@ package buscompose import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // The node seat node-engine (novox/hq ADR 0563 §1): every node's engine holds it on its own bus account, and no module @@ -22,6 +23,16 @@ const ( // EngineActsBucket is that state's bucket. var EngineActsBucket = BucketName(EngineModule, EngineActsState) +// EnginePackagesBucket is the engine's state of its node's declared packages (novox/hq ADR 0565 clause 6): one key, the +// node's name, written by that node's engine alone. +var EnginePackagesBucket = BucketName(EngineModule, delivery.EnginePackagesState) + +// EnginePackagesGrants is what a node's engine publishes to keep its node's key of the packages bucket: the key alone, +// and its read. +func EnginePackagesGrants(node string) []string { + return append([]string{"$KV." + EnginePackagesBucket + "." + node}, readGrants(EnginePackagesBucket, node)...) +} + // engineGrants is what a node's engine is granted to hold its node seat and to ask the operator (ADR 0563 §1, §3): it // answers its own node's verbs (the caller-named subjects are added beside them with every principal's), says the // seat's events about its node, answers discovery for the seat, asks who answers what (the login shell's execute, ADR @@ -41,5 +52,9 @@ func engineGrants(p Principal) (pub, sub []string) { if own, err := NodeKeysGrants(EngineActsBucket, p.Node); err == nil { pub = append(pub, own...) } + // And its node's declared packages, its own key alone (ADR 0565 clause 6). + if safeSubject.MatchString(p.Node) { + pub = append(pub, EnginePackagesGrants(p.Node)...) + } return append(pub, tp...), append(sub, ts...) } diff --git a/go/buscompose/writers.go b/go/buscompose/writers.go index 1d94184..77abcd2 100644 --- a/go/buscompose/writers.go +++ b/go/buscompose/writers.go @@ -248,6 +248,10 @@ var WritersTable = []WriterRow{ Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys}, // The acts a node's engine was asked through its node seat, and its own asks (novox/hq ADR 0563 §2, §3): the // node-engine's declared state, created by the grants part, written by each node's engine under its own keys. + // A node's declared packages (novox/hq ADR 0565 clause 6): its engine's own key, read by pacman's verbs. + {State: "a machine's declared packages", Writer: "the node-engine of the machine, its key `` alone", + KeptIn: "key-value " + EnginePackagesBucket, Others: "read by pacman through `reads`", + Subjects: kvOf(EnginePackagesBucket), Writes: ownNodeKeys}, {State: "a machine's hand acts", Writer: "the node-engine of the machine, under its own keys `.>`, compare-and-set", KeptIn: "key-value " + EngineActsBucket + ", the node-engine's declared state acts", Others: "read through the node seat's verb acts", Subjects: kvOf(EngineActsBucket), diff --git a/go/catalogue/upgrade.go b/go/catalogue/upgrade.go index c8c2857..d498f56 100644 --- a/go/catalogue/upgrade.go +++ b/go/catalogue/upgrade.go @@ -3,6 +3,7 @@ package catalogue import ( "fmt" "strings" + "time" ) // What the mesh does when a module's build moves (novox/hq ADR 0236, extending ADR 0162 §3 and ADR @@ -118,3 +119,46 @@ func (m Manifest) irreplaceable() string { } return "" } + +// PersonPolicy is a person's choice of a module's upgrade policy, as the assigner keeps it in its state `policies`, one +// key per module (novox/hq ADR 0565 clause 7, issue 1953 L5-S2): the policy, why, who as the runtime proved the caller, +// and when. Set and taken back through `mesh-wants.upgrade`, each recorded as a hand act. +type PersonPolicy struct { + V int `json:"v"` + Module string `json:"module"` + Policy string `json:"policy"` + Why string `json:"why"` + By string `json:"by"` + At time.Time `json:"at"` +} + +// PersonPolicyVersion is the version of a person's policy's layout. +const PersonPolicyVersion = 1 + +// Check refuses a choice the assigner could not keep: a policy that is not roll, together or record, one other than +// roll without why, or one with no person. +func (p PersonPolicy) Check() error { + switch { + case p.Module == "": + return fmt.Errorf("a person's policy names its module") + case p.Policy != PolicyRoll && p.Policy != PolicyTogether && p.Policy != PolicyRecord: + return fmt.Errorf("%s: a policy is %q, %q or %q, not %q", p.Module, PolicyRoll, PolicyTogether, PolicyRecord, p.Policy) + case strings.TrimSpace(p.Why) == "": + return fmt.Errorf("%s: a person's policy says why", p.Module) + case strings.TrimSpace(p.By) == "": + return fmt.Errorf("%s: a person's policy names who chose it", p.Module) + } + return nil +} + +// Upgrade is a module's policy and where it comes from (ADR 0565 clause 7): the one derivation every reader takes, +// `mesh-wants.upgrade` and each node agent's judgement of a move alike. A person's choice over the module's word, its +// irreplaceable data and the default; the bus is never rolled, whoever says so (ADR 0236). By names the person when one +// chose. +func Upgrade(m Manifest, chosen *PersonPolicy) (policy, from, why, by string) { + policy, from, why = DerivedUpgrade(m) + if from == FromBus || chosen == nil { + return policy, from, why, "" + } + return chosen.Policy, FromPerson, chosen.Why, chosen.By +} diff --git a/go/catalogue/upgrade_test.go b/go/catalogue/upgrade_test.go index 6e41a73..79cfbff 100644 --- a/go/catalogue/upgrade_test.go +++ b/go/catalogue/upgrade_test.go @@ -32,6 +32,26 @@ func TestWhereAModulesUpgradePolicyComesFrom(t *testing.T) { if policy != c.policy || from != c.from { t.Errorf("%s: %s from %s, want %s from %s", c.name, policy, from, c.policy, c.from) } + // ADR 0565 clause 7 (issue 1953 L5-S3): a person's choice wins over each source but the bus; none leaves the + // derivation as it was. + person := &PersonPolicy{V: PersonPolicyVersion, Module: c.m.Module, Policy: PolicyTogether, Why: "one zone", By: "person.jochen"} + policy, from, _, by := Upgrade(c.m, person) + wantPolicy, wantFrom, wantBy := PolicyTogether, FromPerson, "person.jochen" + if c.from == FromBus { + wantPolicy, wantFrom, wantBy = c.policy, FromBus, "" + } + if policy != wantPolicy || from != wantFrom || by != wantBy { + t.Errorf("%s with a person's choice: %s from %s by %q", c.name, policy, from, by) + } + if policy, from, _, _ := Upgrade(c.m, nil); policy != c.policy || from != c.from { + t.Errorf("%s with no choice: %s from %s", c.name, policy, from) + } + } + for _, p := range []PersonPolicy{{Module: "m", Policy: "sometimes", Why: "x", By: "p"}, {Module: "m", Policy: PolicyRecord, By: "p"}, + {Module: "m", Policy: PolicyRecord, Why: "x"}} { + if p.Check() == nil { + t.Errorf("%+v was accepted", p) + } } } diff --git a/go/delivery/control_records.go b/go/delivery/control_records.go new file mode 100644 index 0000000..9d9f869 --- /dev/null +++ b/go/delivery/control_records.go @@ -0,0 +1,95 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// **The control records' layouts, out of the controller** (novox/hq ADR 0565 clauses 2, 3 and 6; issue 1953 L5-S2): what +// the indexer keeps of the catalogue's predefined list and of each open pull request's config declarations, which +// mesh-config reads in place of the controller's mirrors and `pull-declares`; and each node's declared packages, which +// its engine writes in place of the controller's mirror. + +// The indexer's two states (clause 2). +const ( + // IndexerPredefinedState holds one key, PredefinedListKey: the catalogue's predefined.json at the newest merge taken. + IndexerPredefinedState = "predefined" + PredefinedListKey = "list" + // IndexerConfigCandidatesState holds one key per open pull request (ConfigCandidateKey). + IndexerConfigCandidatesState = "config-candidates" +) + +// PredefinedKept is the indexer's key `predefined.list`: the list whole, as predefined.json holds it, with the merge it +// was read at. A merge older than the one the key names never replaces it (ADR 0348 §7). +type PredefinedKept struct { + V int `json:"v"` + Commit string `json:"commit"` + At time.Time `json:"at"` + List json.RawMessage `json:"list"` +} + +// ConfigDeclaration is one config key a module's manifest declares, in the layout of the controller's declared-config +// mirror (mesh-controller internal/inventory DeclaredMirrored) unchanged, so mesh-config reads the indexer's as it read +// the mirror's. +type ConfigDeclaration struct { + Kind string `json:"kind"` + Default json.RawMessage `json:"default"` + Why string `json:"why"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum json.RawMessage `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Predefined string `json:"predefined,omitempty"` + Required bool `json:"required,omitempty"` + Runtime bool `json:"runtime,omitempty"` +} + +// ConfigCandidate is one open pull request's key of `config-candidates`: its head, and per module the config +// declarations the head's manifests add or change against the trunk's, an empty set when none. Written at each new +// head and each trunk merge that changes a module it names; deleted when the pull request merges or closes. +type ConfigCandidate struct { + V int `json:"v"` + Pull string `json:"pull"` + Head string `json:"head"` + Declares map[string]map[string]ConfigDeclaration `json:"declares"` + At time.Time `json:"at"` +} + +// ConfigCandidateKey is a pull request's key, `owner/repository#number` with `/` and `#` as dots, the form mesh-config +// keeps a pull request's values under. +func ConfigCandidateKey(pull string) (string, error) { + owner, rest, ok := strings.Cut(strings.TrimSpace(pull), "/") + repo, number, ok2 := strings.Cut(rest, "#") + if !ok || !ok2 || owner == "" || repo == "" || number == "" || strings.ContainsAny(owner+repo+number, ".*> ") { + return "", fmt.Errorf("delivery: %q is not a pull request, owner/repository#number", pull) + } + return owner + "." + repo + "." + number, nil +} + +// The node-engine's packages (clause 6): its state `packages`, one key, the node's name, written by that node's engine +// alone. +const EnginePackagesState = "packages" + +// PackageDeclared is one package a node's modules declare, the controller's declared-packages mirror's entry unchanged +// (mesh-controller internal/inventory PackageDeclared): its name, the modules that declare it that way, sorted, and +// whether they declare it absent. +type PackageDeclared struct { + Package string `json:"package"` + Modules []string `json:"modules"` + Absent bool `json:"absent"` +} + +// DeclaredPackages is one node's key of `packages`, the mirror's value unchanged: its packages sorted by name, absent +// after present. Fixed fields and no time, so the same declaration is the same bytes (ADR 0289 §1). +type DeclaredPackages struct { + Node string `json:"node"` + Packages []PackageDeclared `json:"packages"` +} + +// The assigner's state of a person's upgrade policy, one key per module (clause 7; catalogue.PersonPolicy). +const AssignerPoliciesState = "policies" diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go new file mode 100644 index 0000000..6e4d42b --- /dev/null +++ b/go/delivery/control_records_test.go @@ -0,0 +1,45 @@ +package delivery + +import ( + "encoding/json" + "testing" +) + +// ADR 0565 clauses 2 and 6 (issue 1953 L5-S2): the control records' layouts read what their writers write. The +// packages value is the controller's declared-packages mirror's, as its writer's own test pins it (mesh-controller +// internal/inventory/declared_packages_test.go at 0fbb5ef3); the config declaration is its declared-config mirror's +// (DeclaredMirrored); a pull request's key is mesh-config's form. +func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { + for _, c := range []struct { + name string + raw string + into any + }{ + {"declared packages", `{"node":"one","packages":[{"package":"curl","modules":["backup","web"],"absent":false},` + + `{"package":"nano","modules":["web"],"absent":true}]}`, &DeclaredPackages{}}, + {"a config candidate", `{"v":1,"pull":"novox/mesh-catalog#42","head":"abc","declares":{"mail":{"quota":{"kind":"config",` + + `"default":10,"why":"a quota","type":"integer","minimum":1}}},"at":"2026-10-29T12:00:00Z"}`, &ConfigCandidate{}}, + {"the predefined list", `{"v":1,"commit":"abc","at":"2026-10-29T12:00:00Z","list":{"domain":{"type":"string"}}}`, &PredefinedKept{}}, + } { + if err := json.Unmarshal([]byte(c.raw), c.into); err != nil { + t.Fatalf("%s: %v", c.name, err) + } + again, _ := json.Marshal(c.into) + var want, got any + _ = json.Unmarshal([]byte(c.raw), &want) + _ = json.Unmarshal(again, &got) + if string(jsonOf(want)) != string(jsonOf(got)) || (c.name == "declared packages" && string(again) != c.raw) { + t.Errorf("%s reads back as %s", c.name, again) + } + } + for pull, want := range map[string]string{"novox/mesh-catalog#42": "novox.mesh-catalog.42", "x#1": "", "a/b": "", "a/b.c#1": ""} { + if got, err := ConfigCandidateKey(pull); got != want || (want == "") != (err != nil) { + t.Errorf("%s keyed %q, %v", pull, got, err) + } + } +} + +func jsonOf(v any) []byte { + b, _ := json.Marshal(v) + return b +} From bc6c8a6d244ae5d782f188fc825e475ef7170cbd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:14:42 +0100 Subject: [PATCH 20/52] Let a bundle write a key with its own time to live and hear its expiry, so pulses, settles and bounds expire on the bus through the runtime (issue 1952) --- go/seat.go | 6 ++++++ go/state.go | 34 +++++++++++++++++++++++++++++++++- go/state_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/go/seat.go b/go/seat.go index 18f49a5..8a9b71f 100644 --- a/go/seat.go +++ b/go/seat.go @@ -177,6 +177,12 @@ func (s KeptState) Create(key string, value any) (uint64, error) { return s.revisionOf(s.ask("create", map[string]any{"key": key, "value": value})) } +// CreateTTL is Create with the key's own time to live (PutTTL): refused as ErrChanged when the key has a value, so an act +// begun once is not begun again; when ttl passes a watch hands the key over as OpExpire. +func (s KeptState) CreateTTL(key string, value any, ttl time.Duration) (uint64, error) { + return s.revisionOf(s.ask("create", map[string]any{"key": key, "value": value, "ttl_seconds": ttlSeconds(ttl)})) +} + // Update writes a key only when it is still at revision, and returns the new one; a key changed // meanwhile is refused, so of two writers that read one value only the first writes. func (s KeptState) Update(key string, value any, revision uint64) (uint64, error) { diff --git a/go/state.go b/go/state.go index 56490b6..ac3d5fd 100644 --- a/go/state.go +++ b/go/state.go @@ -20,7 +20,18 @@ type StateEntry struct { Revision uint64 `json:"revision"` } -// StateChange is one change a watch hands over: a key put or deleted. +// The operations a watch hands over (StateChange.Op). +const ( + // OpPut is a key written. + OpPut = "put" + // OpDelete is a key deleted or purged. + OpDelete = "delete" + // OpExpire is a key whose own time to live ended (CreateTTL, PutTTL): the runtime says the bus's marker of it as + // "expire", so a watcher tells an expiry, which is the event a pulse or a bound waits for, from a delete. + OpExpire = "expire" +) + +// StateChange is one change a watch hands over: a key put (OpPut), deleted (OpDelete) or expired (OpExpire). type StateChange struct { State string `json:"state"` Key string `json:"key"` @@ -98,6 +109,27 @@ func (s KeptState) Put(key string, value any) (uint64, error) { return put.Revision, json.Unmarshal(raw, &put) } +// PutTTL writes a key whole with its own time to live — the owner's instances only — and returns the revision written: +// the bus removes it when ttl passes and a watch hands that over as OpExpire. The state is declared `key-ttl`. +func (s KeptState) PutTTL(key string, value any, ttl time.Duration) (uint64, error) { + raw, err := s.ask("put", map[string]any{"key": key, "value": value, "ttl_seconds": ttlSeconds(ttl)}) + if err != nil { + return 0, err + } + var put struct { + Revision uint64 `json:"revision"` + } + return put.Revision, json.Unmarshal(raw, &put) +} + +// ttlSeconds is a time to live as the runtime takes it, whole seconds, at least one. +func ttlSeconds(ttl time.Duration) int64 { + if s := int64(ttl / time.Second); s > 1 { + return s + } + return 1 +} + // Delete removes a key — the owner's instances only. A key that was not there is not an error. func (s KeptState) Delete(key string) error { _, err := s.ask("delete", map[string]any{"key": key}) diff --git a/go/state_test.go b/go/state_test.go index 79c5350..7b5f109 100644 --- a/go/state_test.go +++ b/go/state_test.go @@ -3,6 +3,8 @@ package stdio import ( "bufio" "encoding/json" + "errors" + "fmt" "io" "strings" "testing" @@ -33,6 +35,12 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) { return State("servers").Put(a["key"].(string), a["value"]) }}, {Name: "recall", Run: func(a map[string]any) (any, error) { return State("servers").Get(a["key"].(string)) }}, + {Name: "pulse", Run: func(a map[string]any) (any, error) { + if a["create"] == true { + return State("servers").CreateTTL(a["key"].(string), 1, 90*time.Second) + } + return State("servers").PutTTL(a["key"].(string), 1, 90*time.Second) + }}, }, inR, outW) }() lines := bufio.NewScanner(outR) @@ -87,6 +95,15 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) { if err := <-watched; err != nil { t.Fatal(err) } + // 1952 (the runtime's per-key time to live, mesh-tools 1583fa85): an expiry is handed over as its own operation. + send(map[string]any{"jsonrpc": "2.0", "id": 49, "method": "mesh/state", "params": map[string]any{"watch": watch["id"], + "change": map[string]any{"state": "servers", "key": "all.gone", "op": OpExpire}}}) + if r := next(); r["id"] != float64(49) || r["error"] != nil { + t.Fatalf("an expiry was answered %v", r) + } + if s := <-seen; s != "expire all.gone " { + t.Fatalf("handled %q", s) + } // A change for a watch this bundle never asked for is refused. send(map[string]any{"jsonrpc": "2.0", "id": 51, "method": "mesh/state", "params": map[string]any{"watch": "bundle-999", "change": map[string]any{"key": "x", "op": "put"}}}) @@ -105,6 +122,32 @@ func TestABundleKeepsAndWatchesStateThroughTheRuntime(t *testing.T) { if r := next(); r["id"] != float64(2) || r["result"].(map[string]any)["content"].([]any)[0].(map[string]any)["text"] != "9" { t.Fatalf("the put answered %v", r) } + // PutTTL and CreateTTL ask the runtime's put and create with the key's time to live in whole seconds; a create the + // runtime refuses as changed (-32010) is ErrChanged. + for i, create := range []bool{false, true} { + send(map[string]any{"jsonrpc": "2.0", "id": 20 + i, "method": "tools/call", "params": map[string]any{"name": "pulse", + "arguments": map[string]any{"key": "pulse.S1", "create": create}}}) + asked := next() + p := asked["params"].(map[string]any) + want := map[bool]string{false: "mesh/state.put", true: "mesh/state.create"}[create] + if asked["method"] != want || p["ttl_seconds"] != float64(90) || p["key"] != "pulse.S1" { + t.Fatalf("asked %v", asked) + } + if create { + send(map[string]any{"jsonrpc": "2.0", "id": asked["id"], "error": map[string]any{"code": CodeChanged, "message": "exists"}}) + if r := next(); r["id"] != float64(21) || !strings.Contains(fmt.Sprint(r), "exists") { + t.Fatalf("a refused create answered %v", r) + } + continue + } + send(map[string]any{"jsonrpc": "2.0", "id": asked["id"], "result": map[string]any{"revision": 4}}) + if r := next(); r["id"] != float64(20) { + t.Fatalf("the put answered %v", r) + } + } + if !errors.Is(&refused{code: CodeChanged, message: "x"}, ErrChanged) { + t.Fatal("a refusal coded -32010 is not ErrChanged") + } // Get of a key with no value is nil. send(map[string]any{"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": map[string]any{"name": "recall", "arguments": map[string]any{"key": "nothing"}}}) From 2e5cc4f16635a1705596be9b9c6b61d5050a424b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:17:50 +0100 Subject: [PATCH 21/52] Let a module's log keep its entries for an age, so the hand-act log holds each act 400 days as ADR 0565 decides instead of refusing when full (issue 1953 S-age) --- go/buscompose/buscompose_test.go | 5 +++++ go/buscompose/log.go | 7 +++++-- go/catalogue/logs.go | 25 ++++++++++++++++++++----- go/catalogue/logs_test.go | 13 +++++++++++-- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 56d0879..4e1d0d2 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -659,6 +659,11 @@ func healthChecks(t *testing.T) { t.Errorf("%s writes the assigner's policies", u.Username()) } } + // 1953 S-age (ADR 0565 §10): a log that says an age keeps its entries that long and no longer; one that says none, as + // long as the log (ADR 0297). + if aged, kept := (Log{Module: "mesh-hand-acts", Name: "acts", MaxAgeDays: 400}).Config(), (Log{Module: "m", Name: "l"}).Config(); aged.MaxAge != 400*24*time.Hour || kept.MaxAge != 0 || !aged.DenyDelete { + t.Errorf("an aged log's stream keeps %s, an unaged one %s", aged.MaxAge, kept.MaxAge) + } // The tools account's secret is the module's own broker-, which its manifest's own-secrets names and the // vault keeps by that name (AccountSecret). if w := (WantedAccount{User: "a.nats.nats-tools", Kind: KindProcess, Node: "a", Module: "nats", Process: "nats-tools", diff --git a/go/buscompose/log.go b/go/buscompose/log.go index 5c22001..9f1370f 100644 --- a/go/buscompose/log.go +++ b/go/buscompose/log.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/nats-io/nats.go/jetstream" ) @@ -37,6 +38,8 @@ type Log struct { Name string // MaxMiB is its cap in MiB; zero is LogDefaultMiB. MaxMiB int + // MaxAgeDays is how long an entry is kept; zero is as long as the log (catalogue.LogDeclaration). + MaxAgeDays int } // LogStreamPrefix starts every log's stream name, which no other stream of the mesh's starts with. @@ -74,7 +77,7 @@ func (l Log) Why() string { } // Config is the stream a log is, field by field as novox/hq ADR 0297 fixes it: a file stream kept by -// limits, with no maximum age and no cap per key, whose message is an entry's value and 4 KiB of +// limits, with no maximum age unless the log says one (ADR 0565 §10) and no cap per key, whose message is an entry's value and 4 KiB of // headers at most, that refuses a new entry when full rather than // drop an old one, and that refuses deleting an entry or purging it. Direct gets are allowed, which // is how the runtime reads it. @@ -85,7 +88,7 @@ func (l Log) Config() jetstream.StreamConfig { Subjects: []string{l.Subject() + ".>"}, Storage: jetstream.FileStorage, Retention: jetstream.LimitsPolicy, - MaxAge: 0, + MaxAge: time.Duration(l.MaxAgeDays) * 24 * time.Hour, MaxMsgs: -1, MaxMsgsPerSubject: -1, MaxBytes: l.MaxBytes(), diff --git a/go/catalogue/logs.go b/go/catalogue/logs.go index 7856b8c..d20aa1c 100644 --- a/go/catalogue/logs.go +++ b/go/catalogue/logs.go @@ -27,8 +27,15 @@ type LogDeclaration struct { // MaxMiB is the log's cap in MiB; zero is LogDefaultMiB. When full, the log refuses new entries // and never drops old ones. MaxMiB int `json:"max-mib,omitempty"` + // MaxAgeDays is how long an entry is kept, in days; zero is as long as the log (novox/hq ADR 0297). The one way an + // entry leaves a log: the server ages it out, never a delete or a purge (novox/hq ADR 0565 §10: the hand-act log + // keeps every act 400 days; issue 1953 S-age). + MaxAgeDays int `json:"max-age-days,omitempty"` } +// LogMostAgeDays is the longest a log may keep an entry by age: past it, a log keeps its entries as long as it is. +const LogMostAgeDays = 3650 + // Cap is the log's cap in MiB, the default where none is said. func (l LogDeclaration) Cap() int { if l.MaxMiB == 0 { @@ -37,20 +44,21 @@ func (l LogDeclaration) Cap() int { return l.MaxMiB } -// UnmarshalJSON reads a log as its bare name, or as {name, max-mib}. +// UnmarshalJSON reads a log as its bare name, or as {name, max-mib, max-age-days}. func (l *LogDeclaration) UnmarshalJSON(raw []byte) error { trimmed := bytes.TrimSpace(raw) if len(trimmed) > 0 && trimmed[0] == '"' { return json.Unmarshal(trimmed, &l.Name) } var full struct { - Name string `json:"name"` - MaxMiB *int `json:"max-mib"` + Name string `json:"name"` + MaxMiB *int `json:"max-mib"` + MaxAgeDays *int `json:"max-age-days"` } dec := json.NewDecoder(bytes.NewReader(trimmed)) dec.DisallowUnknownFields() if err := dec.Decode(&full); err != nil { - return fmt.Errorf("a log is either a name or {name, max-mib}: %w", typedUnknown(err)) + return fmt.Errorf("a log is either a name or {name, max-mib, max-age-days}: %w", typedUnknown(err)) } l.Name = full.Name l.MaxMiB = 0 @@ -61,12 +69,19 @@ func (l *LogDeclaration) UnmarshalJSON(raw []byte) error { } l.MaxMiB = *full.MaxMiB } + l.MaxAgeDays = 0 + if full.MaxAgeDays != nil { + if *full.MaxAgeDays < 1 || *full.MaxAgeDays > LogMostAgeDays { + return fmt.Errorf("log %q: max-age-days is between 1 and %d, not %d", full.Name, LogMostAgeDays, *full.MaxAgeDays) + } + l.MaxAgeDays = *full.MaxAgeDays + } return nil } // MarshalJSON writes back the short form when there is nothing else to say. func (l LogDeclaration) MarshalJSON() ([]byte, error) { - if l.MaxMiB == 0 { + if l.MaxMiB == 0 && l.MaxAgeDays == 0 { return json.Marshal(l.Name) } type plain LogDeclaration diff --git a/go/catalogue/logs_test.go b/go/catalogue/logs_test.go index 76e7ee0..aeea053 100644 --- a/go/catalogue/logs_test.go +++ b/go/catalogue/logs_test.go @@ -24,7 +24,15 @@ func TestAManifestMaySayWhatLogsItKeeps(t *testing.T) { if string(out) != `["changes",{"name":"moves","max-mib":2048}]` { t.Fatalf("written back as %s", out) } - for _, edge := range []string{`{"name":"a","max-mib":1}`, `{"name":"a","max-mib":8192}`} { + // 1953 S-age (ADR 0565 §10): a log may keep its entries for an age, written back with it. + aged, err := ParseManifest([]byte(`{"module":"mesh-hand-acts","version":"1","logs":[{"name":"acts","max-age-days":400}]}`)) + if err != nil || aged.Logs[0].MaxAgeDays != 400 { + t.Fatalf("an aged log read as %+v, %v", aged.Logs, err) + } + if out, _ := json.Marshal(aged.Logs); string(out) != `[{"name":"acts","max-age-days":400}]` { + t.Fatalf("an aged log written back as %s", out) + } + for _, edge := range []string{`{"name":"a","max-mib":1}`, `{"name":"a","max-mib":8192}`, `{"name":"a","max-age-days":3650}`} { if _, err := ParseManifest([]byte(`{"module":"a","version":"1","logs":[` + edge + `]}`)); err != nil { t.Errorf("%s is inside the caps and was refused: %v", edge, err) } @@ -42,7 +50,8 @@ func TestALogIsNamedLocallyAndCapped(t *testing.T) { {`{"module":"a","version":"1","logs":[{"name":"c","max-mib":8193}]}`, `between 1 and 8192 MiB`}, {`{"module":"a","version":"1","logs":[{"name":"c","max-mib":-1}]}`, `between 1 and 8192 MiB`}, {`{"module":"a","version":"1","logs":[{"name":"c","max-mib":0}]}`, `max-mib is between 1 and 8192, not 0`}, - {`{"module":"a","version":"1","logs":[{"name":"c","stream":"LOG_x"}]}`, `{name, max-mib}`}, + {`{"module":"a","version":"1","logs":[{"name":"c","stream":"LOG_x"}]}`, `{name, max-mib, max-age-days}`}, + {`{"module":"a","version":"1","logs":[{"name":"c","max-age-days":0}]}`, `max-age-days is between 1 and 3650, not 0`}, {`{"module":"a.b","version":"1","logs":["c"]}`, `no dot`}, } { _, err := ParseManifest([]byte(c.manifest)) From 2d2aa7ba9bbd47e92132a5a38f8a2aea6bea0753 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:21:12 +0100 Subject: [PATCH 22/52] =?UTF-8?q?Carry=20the=20catalogue's=20predefined=20?= =?UTF-8?q?list=20on=20the=20bridge's=20merged=20message,=20so=20the=20ind?= =?UTF-8?q?exer=20keeps=20it=20without=20the=20controller's=20mirror=20(is?= =?UTF-8?q?sue=201953=20L5-D1,=20ADR=200565=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/forge_inputs_test.go | 19 +++++++++++++++++++ go/delivery/source.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/go/delivery/forge_inputs_test.go b/go/delivery/forge_inputs_test.go index 5238339..4b76ee1 100644 --- a/go/delivery/forge_inputs_test.go +++ b/go/delivery/forge_inputs_test.go @@ -33,6 +33,22 @@ func TestForgeMergedCarriesManifestsAndLocks(t *testing.T) { if got.Manifests["a"] != oddManifest || !SameForge(got, withV(f)) { t.Errorf("the message changed on the way: %q", got.Manifests["a"]) } + // ADR 0565 §2 (issue 1953 D1): a merged message carries the predefined list exact, or says the merge holds none; a + // pull message carries neither. + f.Predefined = []byte(`{"domain":{"type":"string"}}`) + if data, err := EncodeForge(f); err != nil { + t.Fatal(err) + } else if got, err := DecodeForge(data); err != nil || string(got.Predefined) != `{"domain":{"type":"string"}}` { + t.Errorf("the predefined list read back as %s, %v", got.Predefined, err) + } + f.Predefined, f.PredefinedRead = nil, true + if _, err := EncodeForge(f); err != nil { + t.Errorf("a merge holding no predefined list: %v", err) + } + pull := Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true} + if _, err := EncodeForge(pull); err == nil || !strings.Contains(err.Error(), "only a merged message") { + t.Errorf("a pull message carrying the predefined list: %v", err) + } } func withV(f Forge) Forge { f.V = Version; return f } @@ -50,6 +66,9 @@ func TestForgeMergedRefusesBadInputs(t *testing.T) { // No repository holds placement (ADR 0567 §11): a message carrying placement files is refused, on a merge or a pull. "placement files": {func(f *Forge) { f.RetiredPlacements = []byte(`{"anchor":"{}"}`) }, "mesh-placement"}, "placements read": {func(f *Forge) { f.RetiredRead = true }, "mesh-placement"}, + // ADR 0565 §2 (issue 1953 D1): the predefined list is one JSON object, said present or absent, never both. + "predefined not an object": {func(f *Forge) { f.Predefined = []byte(`["domain"]`) }, "one JSON object"}, + "predefined both": {func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true }, "says the merge holds none"}, } { f := mergedWithInputs() tc.mut(&f) diff --git a/go/delivery/source.go b/go/delivery/source.go index 6b0d25c..3c09503 100644 --- a/go/delivery/source.go +++ b/go/delivery/source.go @@ -542,6 +542,31 @@ type Forge struct { // refused, loudly, rather than read with them dropped. RetiredPlacements json.RawMessage `json:"placements,omitempty"` RetiredRead bool `json:"placements-read,omitempty"` + // Predefined is the catalogue's predefined.json, exact, at the merge commit of a merged message (novox/hq ADR 0565 + // §2): what the indexer keeps as its state `predefined`. PredefinedRead says the bridge read the merge and it holds + // no predefined.json. Carried on a merged message only; an older bridge sends neither, and the indexer keeps what it + // held. + Predefined json.RawMessage `json:"predefined,omitempty"` + PredefinedRead bool `json:"predefined-read,omitempty"` +} + +// checkPredefined refuses the predefined list on anything but a merged message, a list that is not one JSON object, and +// a list said both present and absent. +func (f Forge) checkPredefined() error { + if len(f.Predefined) == 0 && !f.PredefinedRead { + return nil + } + if f.Kind != ForgeMerged { + return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind) + } + if len(f.Predefined) > 0 && f.PredefinedRead { + return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)") + } + var list map[string]json.RawMessage + if len(f.Predefined) > 0 && json.Unmarshal(f.Predefined, &list) != nil { + return fmt.Errorf("delivery: the predefined list a merged message carries is not one JSON object") + } + return nil } func (f Forge) check() error { @@ -558,6 +583,9 @@ func (f Forge) check() error { return fmt.Errorf("delivery: a forge message carries placement files, which no repository holds since ADR 0567: " + "placement is the seat mesh-placement's state") } + if err := f.checkPredefined(); err != nil { + return err + } if err := checkRepository(f.Repository); err != nil { return err } From 4204afd2400fddf0c92524f1801c2b352ef404dd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:28:45 +0100 Subject: [PATCH 23/52] Name mesh-config's compare, take-over and hand-back on its seat, so callers reach the config switch ADR 0565 decides by the seat (issue 1953) --- go/catalogue/seats.go | 14 +++++++++++++- go/catalogue/seats_test.go | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 1b70e38..7f39dcf 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -245,13 +245,25 @@ var defaultSeats = append([]Seat{ // The owner of every config value (novox/hq ADR 0348 §9, design 57 K3): its holder alone writes the buckets // mesh-config_values, _resolved, _predefined and _proposals, and serves the one verb `config` (§1, §10). // Optional until the mesh-config that runs serves it (design 33 §7). - {Name: "mesh-config", Scope: ScopeMesh, Decision: "novox/hq ADR 0348", + {Name: "mesh-config", Scope: ScopeMesh, Decision: "novox/hq ADR 0348, ADR 0565", Serves: []Verb{ {Name: "config", Optional: true, Description: "Set, clear, list or read the history of config values: " + "mesh config or node config for a predefined key, or a module's own value on one node, each set " + "with why, validated, and naming each module on each node whose value moves. A trusted value only " + "from the controller's terminal.", Input: configInput()}, + // The switch of config to its own sources (novox/hq ADR 0565 §4): optional until every mesh-config serves it. + {Name: "compare", Optional: true, Description: "Every difference between config's own sources and the " + + "controller's records, each with its reason: a module's declared config against the controller's mirror, " + + "the indexer's predefined list against the mirror's, and each open pull request's config candidate against " + + "what the controller told. The take-over waits for none.", + Input: schema(map[string]string{}, nil)}, + {Name: "take-over", Optional: true, Description: "Ask the operator's Approve to switch config to its own " + + "sources (the indexer); refused while compare shows a difference.", + Input: schema(map[string]string{"why": "why, in a sentence"}, []string{"why"})}, + {Name: "hand-back", Optional: true, Description: "Switch config back to the controller's mirrors, until the " + + "controller hands its lease away; refused after it.", + Input: schema(map[string]string{"why": "why, in a sentence"}, []string{"why"})}, }}, // The provisioner's server (novox/hq ADR 0349 clause 3, design 57 P4): its holder keeps the registry of which // domain answers which kind of placeholder, and the buckets mesh-provisioner_needs, _applied and _collect, diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index 0226c15..ad001bb 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -71,12 +71,14 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { {WatchdogsSeat, "signals,stuck,presence,dead-letters,resync", ""}, {SelfCheckSeat, "doctor,probes", "checked"}, {HealersSeat, "healers,heal,drill", "healer-acted"}, + // And mesh-config's switch (ADR 0565 §4), beside its config. + {"mesh-config", "config,compare,take-over,hand-back", ""}, // And ADR 0565 clause 1's two. {WantsSeat, "upgrade,bus", "policy-changed"}, {HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"}, } { s, ok := SeatNamed(c.seat) - if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.HasPrefix(s.Decision, "novox/hq ADR 056") || + if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 056") || strings.Join(VerbNames(s.Serves), ",") != c.verbs || strings.Join(s.Emits, ",") != c.emits { t.Errorf("the health seat %s is %v, %+v", c.seat, ok, s) } From 233b6be5ddd15ed274252db429287f47716f1125 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:31:02 +0100 Subject: [PATCH 24/52] Compose each node's provisioning plans from the bus alone, in a Planning mode that reads no secret, so mesh-provisioner composes the plans the controller did (issue 1954 L6-P1, ADR 0570 clause 2) --- go/compose/credentials.go | 8 ++++++ go/compose/frombus/frombus.go | 42 ++++++++++++++++++++++++++++++ go/compose/frombus/frombus_test.go | 18 +++++++++++++ go/compose/render.go | 2 +- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/go/compose/credentials.go b/go/compose/credentials.go index 9766887..8d39bbd 100644 --- a/go/compose/credentials.go +++ b/go/compose/credentials.go @@ -23,6 +23,10 @@ const ( // Foreseeing composes an own secret the next send would make with a stand-in and names it foreseen, while one the send would // refuse (a bus credential nobody issued, a given secret under an old key) is refused here, in the same words. Foreseeing + // Planning composes for a provisioning plan alone (novox/hq ADR 0570 clause 2): every own secret the Input holds none + // for stands in, named foreseen, and nothing is refused for a value, since a plan names secrets and never carries one. + // The seat mesh-provisioner's holder composes so, reading no secret; a rendering in this mode is never applied. + Planning ) // ForeseenCertificate and ForeseenBusUsers stand for what the Input does not carry, in a composition asked ahead of the send. @@ -344,6 +348,10 @@ func (c *composer) ownSecretsFor(node Node, plan catalogue.Resolution, handing c continue } sealed, held := c.moduleSecretIfIssued(node, m.Module, name) + if !held && mode == Planning { + mark(m.Module, name) + continue + } if !held && mode == Foreseeing { // Asked ahead of the send: what the send would do about it, by the send's own rules. Made by it -- a stand-in, // named; refused by it -- refused here, the same words (novox/hq issue 275). diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index d8dd9f7..8478333 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -426,6 +426,48 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s return out, skipped, held, err } +// Plans is the provisioning plan of each module handed to the provisioner on the scope's node, composed from the bus +// alone (novox/hq ADR 0570 clause 2): the controller's handedPlans over the same input (mesh-control +// cmd/mesh-controller/compose_sdk.go), its rendering in the Foreseeing mode with the mesh's generators. A plan names +// templates, files, own secrets by name, items, collections and provisions, never a value, so no value is read for it: +// the seat mesh-provisioner's holder composes every node's plans with a Reader that reads no secret. +func Plans(ctx context.Context, r Reader, mesh Mesh, s Scope) (map[string]catalogue.Plan, error) { + in, err := Input(ctx, r, mesh, s) + if err != nil { + return nil, err + } + return PlansOf(in, s.Node) +} + +// PlansOf is Plans over an input already filled, rendered in the Planning mode (compose.Planning): every own secret the +// input holds none for stands in, so no value and no record of one is needed. +func PlansOf(in compose.Input, node string) (map[string]catalogue.Plan, error) { + return plansIn(in, node, compose.Planning) +} + +// plansIn is the controller's handedPlans in a mode: Foreseeing is the controller's own. +func plansIn(in compose.Input, node string, mode compose.Mode) (map[string]catalogue.Plan, error) { + resolved, err := compose.Resolve(in, node) + if err != nil { + return nil, fmt.Errorf("frombus: the plans of %s: %w", node, err) + } + rendered, err := compose.Render(in, node, mode) + if err != nil { + return nil, fmt.Errorf("frombus: the plans of %s: %w", node, err) + } + gens, err := compose.Generators(in) + if err != nil { + return nil, fmt.Errorf("frombus: the plans of %s: %w", node, err) + } + with := rendered.Rendering + with.Generators = gens + composed, err := resolved.Plan.Compose(with) + if err != nil { + return nil, fmt.Errorf("frombus: the plans of %s: %w", node, err) + } + return composed.Plans, nil +} + // parser reads a manifest as the indexer does, against the mesh's predefined keys when given. func parser(mesh Mesh) func([]byte) (catalogue.Manifest, error) { return func(raw []byte) (catalogue.Manifest, error) { diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index cd3d74d..69d6333 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -694,6 +694,24 @@ func TestInput(t *testing.T) { wj, _ := json.MarshalIndent(fromStoreMoved, "", " ") t.Errorf("composed from the build differs from the controller's take-in:\n got %s\nwant %s", gj, wj) } + // ADR 0570 clause 2 (issue 1954 L6-P1): the seat mesh-provisioner's plans, composed from the bus alone and with no + // secret read (the Planning mode), are the controller's handedPlans over its own input, which holds its secrets + // (compose_sdk.go: Resolve, Render in Foreseeing, Generators, Plan.Compose), and restic, handed, has one. + noSecrets := movedMesh(true) + noSecrets.own = nil + seat, err := Plans(ctx, noSecrets, Mesh{}, Scope{Node: "n1"}) + if err != nil { + t.Fatalf("the plans from the bus: %v", err) + } + controllers, err := plansIn(storeIn, "n1", compose.Foreseeing) + if err != nil { + t.Fatal(err) + } + if _, handed := seat["restic"]; !handed || !reflect.DeepEqual(seat, controllers) { + gj, _ := json.MarshalIndent(seat, "", " ") + wj, _ := json.MarshalIndent(controllers, "", " ") + t.Errorf("the seat's plans differ from the controller's:\n got %s\nwant %s", gj, wj) + } // And what a provider on another node serves from its settings (mailu's smtp domain, issue 1126: shanks refused // '${setting:domain} ... nothing sets "domain"'): read for that provider's served keys alone, through Served. diff --git a/go/compose/render.go b/go/compose/render.go index e8dca30..66d9814 100644 --- a/go/compose/render.go +++ b/go/compose/render.go @@ -71,7 +71,7 @@ func (c *composer) render(nodeName string, mode Mode) (Rendered, error) { return Rendered{}, err } r.Needed = needed - if mode == Foreseeing || len(foreseenOwn) > 0 { + if mode == Foreseeing || mode == Planning || len(foreseenOwn) > 0 { r.Foreseen = foreseenOwn } var names []string From 4a967933b3686f264a998ec6b792e430a5ceda5c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:32:15 +0100 Subject: [PATCH 25/52] Lay out a provision as a key of mesh-provisioner holding a reference and never a value, and a plan as a key of its own, so the holder, the vault, the clients and the providers read one record (issue 1954 L6-P2, ADR 0570 clauses 1, 2) --- go/catalogue/upgrade_test.go | 5 +- go/provisions/provisions.go | 220 +++++++++++++++++++++++++++++++ go/provisions/provisions_test.go | 78 +++++++++++ 3 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 go/provisions/provisions.go create mode 100644 go/provisions/provisions_test.go diff --git a/go/catalogue/upgrade_test.go b/go/catalogue/upgrade_test.go index 79cfbff..e007897 100644 --- a/go/catalogue/upgrade_test.go +++ b/go/catalogue/upgrade_test.go @@ -53,10 +53,7 @@ func TestWhereAModulesUpgradePolicyComesFrom(t *testing.T) { t.Errorf("%+v was accepted", p) } } -} - -// A policy other than roll says why, and an unknown one is refused. -func TestAnUpgradePolicySaysWhy(t *testing.T) { + // A policy other than roll says why, and an unknown one is refused. for _, u := range []UpgradePolicy{{Policy: PolicyRecord}, {Policy: PolicyTogether}, {Policy: "sometimes", Why: "x"}} { if problems := (Manifest{Module: "m", Upgrade: &u}).upgradeProblems(); len(problems) == 0 { t.Errorf("%+v was accepted", u) diff --git a/go/provisions/provisions.go b/go/provisions/provisions.go new file mode 100644 index 0000000..4981dd4 --- /dev/null +++ b/go/provisions/provisions.go @@ -0,0 +1,220 @@ +// Package provisions is the contract of the seat mesh-provisioner's records (novox/hq ADR 0570 clauses 1, 2 and 5; issue +// 1954): every provision one key of the state `provisions`, holding a reference and never a value, and every handed +// module's provisioning plan one key of the state `plans`. The module mesh-provisioner alone writes both; the vault, +// each node's provisioner client and every provider read them through `reads`. +package provisions + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The seat, its holder and its two states, and the names a reader declares in its `reads`. +const ( + Seat = "mesh-provisioner" + Module = "mesh-provisioner" + ProvisionsState = "provisions" + PlansState = "plans" + ProvisionsRead = Module + "." + ProvisionsState + PlansRead = Module + "." + PlansState + // ProvisionsBucket and PlansBucket are the states' buckets. + ProvisionsBucket = Module + "_" + ProvisionsState + PlansBucket = Module + "_" + PlansState +) + +// The states of a provision (clause 1). +const ( + // Asked is a provision its consumer needs that its provider has not answered yet, or whose dependencies are not + // all provided (clause 3). + Asked = "asked" + // Provided is a provision the provider said it made: set only from the provider's outcome (clause 3). + Provided = "provided" + // Waits is a provision only a person can give, naming what is awaited (clause 7). + Waits = "waits" + // Failed is a provision the provider could not make, with why. + Failed = "failed" + // Retiring is a provision no want needs, asked of the operator before its credential goes (clause 5). + Retiring = "retiring" +) + +// States is every state a provision may be in. +var States = []string{Asked, Provided, Waits, Failed, Retiring} + +// Version is the version of the records' layout. +const Version = 1 + +// token is one part of a key: what a bucket's key takes, no dot. +var token = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// Key is a provision's key: `..`, and `.` where the consumer keeps several values of +// one provision (ADR 0094). The same key the provider's own entry for that consumer has (provider.Entry.Key). +func Key(node, consumer, provision, local string) (string, error) { + parts := []string{node, consumer, provision} + if local != "" { + parts = append(parts, local) + } + for _, p := range parts { + if !token.MatchString(p) { + return "", fmt.Errorf("provisions: %q cannot be part of a provision's key", p) + } + } + return strings.Join(parts, "."), nil +} + +// ParseKey is the inverse of Key. +func ParseKey(key string) (node, consumer, provision, local string, err error) { + parts := strings.Split(key, ".") + if len(parts) < 3 || len(parts) > 4 { + return "", "", "", "", fmt.Errorf("provisions: %q is not ..[.]", key) + } + for _, p := range parts { + if !token.MatchString(p) { + return "", "", "", "", fmt.Errorf("provisions: %q is not ..[.]", key) + } + } + if len(parts) == 4 { + local = parts[3] + } + return parts[0], parts[1], parts[2], local, nil +} + +// PlanKey is a plan's key in `plans`: `.` (clause 2). +func PlanKey(node, module string) (string, error) { + if !token.MatchString(node) || !token.MatchString(module) { + return "", fmt.Errorf("provisions: %s.%s is not a plan's key, .", node, module) + } + return node + "." + module, nil +} + +// End is one end of a provision: a module on a node. +type End struct { + Module string `json:"module"` + Node string `json:"node"` +} + +// Reference is a value the vault keeps, by its id and the generation of the value: never the value, sealed or not +// (ADR 0349 §7). The holder writes the generation the vault says in its own entry (clause 6). +type Reference struct { + ID string `json:"id"` + Generation int `json:"generation"` +} + +// Warrant is the operator's answer to a retiring provision's ask, as the holder writes it into the key (clause 5). +type Warrant struct { + Ask string `json:"ask"` + Option string `json:"option"` + By string `json:"by"` + At time.Time `json:"at"` +} + +// Provision is one key of `provisions`. +type Provision struct { + V int `json:"v"` + Provision string `json:"provision"` + Local string `json:"local,omitempty"` + Consumer End `json:"consumer"` + Provider End `json:"provider"` + // DependsOn are the keys of the provisions this one waits for (clause 3): a database user whose password is the + // vault's. + DependsOn []string `json:"depends-on,omitempty"` + State string `json:"state"` + // Why is the reason for waits, failed and retiring, and for asked when a dependency holds it. + Why string `json:"why,omitempty"` + Reference *Reference `json:"reference,omitempty"` + // Ask is a retiring provision's ask of the operator; Warrant the answer, once given. + Ask string `json:"ask,omitempty"` + Warrant *Warrant `json:"warrant,omitempty"` + At time.Time `json:"at"` +} + +// Key is the provision's key. +func (p Provision) Key() (string, error) { + return Key(p.Consumer.Node, p.Consumer.Module, p.Provision, p.Local) +} + +func (p Provision) check() error { + if _, err := p.Key(); err != nil { + return err + } + if p.Provider.Module == "" || p.Provider.Node == "" { + return fmt.Errorf("provisions: %s names no provider", p.Provision) + } + known := false + for _, s := range States { + known = known || s == p.State + } + switch { + case !known: + return fmt.Errorf("provisions: a provision's state is one of %s, not %q", strings.Join(States, ", "), p.State) + case (p.State == Waits || p.State == Failed || p.State == Retiring) && strings.TrimSpace(p.Why) == "": + return fmt.Errorf("provisions: a %s provision says why", p.State) + case p.Warrant != nil && p.State != Retiring: + return fmt.Errorf("provisions: only a retiring provision carries the operator's warrant") + } + for _, d := range p.DependsOn { + if _, _, _, _, err := ParseKey(d); err != nil { + return err + } + } + return nil +} + +// Encode is p as JSON at Version; it refuses what a holder may not write. +func Encode(p Provision) ([]byte, error) { + p.V = Version + if err := p.check(); err != nil { + return nil, err + } + return json.Marshal(p) +} + +// Decode reads a key's value strictly: a field the layout does not name is refused, so no value, sealed or not, rides in +// a provision (clause 1, ADR 0201 §13). +func Decode(data []byte) (Provision, error) { + var p Provision + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return Provision{}, fmt.Errorf("provisions: a provision is not readable: %w", err) + } + if p.V != Version { + return Provision{}, fmt.Errorf("provisions: a provision of version %d, and this reads %d", p.V, Version) + } + return p, p.check() +} + +// Plan is one key of `plans`: a module's provisioning plan on a node, never a value (clause 2). +type Plan struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Plan catalogue.Plan `json:"plan"` +} + +// EncodePlan is p as JSON at Version. +func EncodePlan(p Plan) ([]byte, error) { + p.V = Version + if _, err := PlanKey(p.Node, p.Module); err != nil { + return nil, err + } + return json.Marshal(p) +} + +// DecodePlan reads a plan's key. +func DecodePlan(data []byte) (Plan, error) { + var p Plan + if err := json.Unmarshal(data, &p); err != nil { + return Plan{}, fmt.Errorf("provisions: a plan is not readable: %w", err) + } + if p.V != Version { + return Plan{}, fmt.Errorf("provisions: a plan of version %d, and this reads %d", p.V, Version) + } + _, err := PlanKey(p.Node, p.Module) + return p, err +} diff --git a/go/provisions/provisions_test.go b/go/provisions/provisions_test.go new file mode 100644 index 0000000..294ef2f --- /dev/null +++ b/go/provisions/provisions_test.go @@ -0,0 +1,78 @@ +package provisions + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// ADR 0570 clauses 1, 2 and 5 (issue 1954 L6-P2): a provision's key is its consumer's node and module, the provision and +// its local name, the provider's entry's key; its value round-trips, names its state and why, and refuses any field the +// layout does not name, so no value rides in it; a plan's key is its node and module. +func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { + at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + db := Provision{Provision: "postgres-database", Consumer: End{Module: "keycloak", Node: "novox"}, + Provider: End{Module: "postgres", Node: "novox"}, DependsOn: []string{"novox.keycloak.secret.db"}, State: Asked, + Reference: &Reference{ID: "provision/postgres/postgres-database", Generation: 2}, At: at} + for _, c := range []struct { + name string + edit func(*Provision) + key string + refused string + }{ + {name: "asked, depending on the vault's", key: "novox.keycloak.postgres-database"}, + {name: "under a local name", edit: func(p *Provision) { p.Local = "replica" }, key: "novox.keycloak.postgres-database.replica"}, + {name: "retiring with the operator's warrant", edit: func(p *Provision) { + p.State, p.Why, p.Ask = Retiring, "keycloak left novox", "a-1" + p.Warrant = &Warrant{Ask: "a-1", Option: "yes", By: "the operator", At: at} + }, key: "novox.keycloak.postgres-database"}, + {name: "an unknown state", edit: func(p *Provision) { p.State = "done" }, refused: "one of"}, + {name: "failed without why", edit: func(p *Provision) { p.State = Failed }, refused: "says why"}, + {name: "a warrant on a provided one", edit: func(p *Provision) { p.State, p.Warrant = Provided, &Warrant{Ask: "a"} }, refused: "only a retiring"}, + {name: "no provider", edit: func(p *Provision) { p.Provider = End{} }, refused: "no provider"}, + {name: "a dotted consumer", edit: func(p *Provision) { p.Consumer.Module = "a.b" }, refused: "cannot be part"}, + {name: "a dependency that is no key", edit: func(p *Provision) { p.DependsOn = []string{"x"} }, refused: "is not"}, + } { + p := db + if c.edit != nil { + c.edit(&p) + } + raw, err := Encode(p) + if c.refused != "" { + if err == nil || !strings.Contains(err.Error(), c.refused) { + t.Errorf("%s: %v, want refused saying %q", c.name, err, c.refused) + } + continue + } + back, err := Decode(raw) + key, kerr := back.Key() + if err != nil || kerr != nil || key != c.key || back.V != Version || back.Reference.Generation != 2 { + t.Errorf("%s: read back as %+v under %q, %v %v", c.name, back, key, err, kerr) + } + if node, consumer, provision, local, err := ParseKey(key); err != nil || node != "novox" || consumer != "keycloak" || + provision != "postgres-database" || local != p.Local { + t.Errorf("%s: %s parses as %s %s %s %s, %v", c.name, key, node, consumer, provision, local, err) + } + } + // A value never rides in a provision: a field the layout does not name is refused. + if _, err := Decode([]byte(`{"v":1,"provision":"p","consumer":{"module":"c","node":"n"},"provider":{"module":"p","node":"n"},` + + `"state":"provided","at":"2026-10-29T22:00:00Z","password":"hunter2"}`)); err == nil { + t.Error("a provision carrying a password was read") + } + plan := Plan{Node: "novox", Module: "restic", Plan: catalogue.Plan{}} + raw, err := EncodePlan(plan) + if err != nil { + t.Fatal(err) + } + if back, err := DecodePlan(raw); err != nil || back.Node != "novox" || back.Module != "restic" { + t.Errorf("a plan read back as %+v, %v", back, err) + } + if k, err := PlanKey("novox", "restic"); k != "novox.restic" || err != nil { + t.Errorf("a plan's key is %q, %v", k, err) + } + if ProvisionsBucket != "mesh-provisioner_provisions" || PlansRead != "mesh-provisioner.plans" { + t.Errorf("the states are %s and %s", ProvisionsBucket, PlansRead) + } +} From 56514aa4a2caa63c4bba3d3c1daba67720b0017e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:37:11 +0100 Subject: [PATCH 26/52] =?UTF-8?q?Add=20the=20assigner's=20eighth=20rule,?= =?UTF-8?q?=20every=20provision=20the=20build=20requires=20provided,=20so?= =?UTF-8?q?=20a=20want=20waits=20on=20its=20provisions=20as=20ADR=200570?= =?UTF-8?q?=20=C2=A74=20decides=20(issue=201954=20D1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/delivery/assigner.go | 7 +++++-- go/delivery/want_composed_test.go | 11 +++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go/delivery/assigner.go b/go/delivery/assigner.go index d5207bd..5587977 100644 --- a/go/delivery/assigner.go +++ b/go/delivery/assigner.go @@ -141,11 +141,14 @@ const ( RuleProviders = 5 // every provider the module requires runs healthy on that node, or at its canary RuleGrant = 6 // the module's credential for F exists (grants..) RuleComposed = 7 // the module's composition for that node at F exists (mesh-controller_composed, ADR 0539 §6) + // RuleProvisions: every provision the module requires on that node is provided (novox/hq ADR 0570 §4): a want's + // words name the key and its state that hold it, as "the database of keycloak on novox is asked of postgres". + RuleProvisions = 8 ) func checkRule(rule int, of string) error { - if rule < RuleBuildDone || rule > RuleComposed { - return fmt.Errorf("delivery: %s names the rule %d, which is not one of the seven (1 to 7)", of, rule) + if rule < RuleBuildDone || rule > RuleProvisions { + return fmt.Errorf("delivery: %s names the rule %d, which is not one of the eight (1 to 8)", of, rule) } return nil } diff --git a/go/delivery/want_composed_test.go b/go/delivery/want_composed_test.go index 72858db..f206367 100644 --- a/go/delivery/want_composed_test.go +++ b/go/delivery/want_composed_test.go @@ -10,13 +10,16 @@ func hashOf(c byte) string { return "sha256:" + digestOf(c) } // 1943/AC3: a want naming its fingerprint and the digest of the composition it allows round-trips, and so does one // carrying the answer to a failed backup, and from ADR 0547's second milestone one naming the build alone or the yes // a person gave; a digest not of the sha256 -// form, a removed want naming a digest, an unknown backup answer and a rule past the seventh are refused. +// form, a removed want naming a digest, an unknown backup answer and a rule past the eighth are refused. func TestAWantNamesItsComposition(t *testing.T) { for _, w := range []Want{ {FP: fp2of('a'), Composed: hashOf('1')}, {FP: fp2of('a'), Composed: hashOf('1'), Backup: BackupAgain}, {FP: fp2of('a'), Composed: hashOf('1'), Backup: BackupSkip, Words: "the operator: go on without a backup"}, {FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: RuleComposed, Words: "no composition of b for ace yet"}, + // ADR 0570 §4 (issue 1954 D1): held for a provision its new build requires, its words naming the key. + {FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: RuleProvisions, + Words: "the database of keycloak on novox is asked of postgres (novox.keycloak.postgres-database)"}, {Removed: true}, // ADR 0547 §2, §3 (issue 1126): from the second milestone a want names the build alone, and the yes a person gave. {FP: fp2of('a'), Self: true}, @@ -44,7 +47,7 @@ func TestAWantNamesItsComposition(t *testing.T) { {Want{Removed: true, Composed: hashOf('1')}, "removed want names no"}, {Want{FP: fp2of('a'), Composed: hashOf('1'), Backup: "later"}, `backup answer "later"`}, {Want{Removed: true, Backup: BackupAgain}, "removed want names no"}, - {Want{FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: 8}, "rule 8"}, + {Want{FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: 9}, "rule 9"}, } for _, r := range refusals { if _, err := EncodeWant(r.w); err == nil || !strings.Contains(err.Error(), r.why) { @@ -54,8 +57,8 @@ func TestAWantNamesItsComposition(t *testing.T) { if _, err := DecodeWant([]byte(`{"v":1,"fp":"` + fp2of('a') + `"}`)); err == nil || !strings.Contains(err.Error(), "names no composition") { t.Errorf("a want of release 5, naming no composition and not the node's own, was read: %v", err) } - if RuleComposed != 7 { - t.Errorf("RuleComposed = %d, want the seventh rule", RuleComposed) + if RuleComposed != 7 || RuleProvisions != 8 { + t.Errorf("RuleComposed = %d and RuleProvisions = %d, want the seventh and the eighth", RuleComposed, RuleProvisions) } } From 6e39ed1852b2571b476df499672c9fffc146f395 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:37:41 +0100 Subject: [PATCH 27/52] Refuse a want's rule past the eighth in the refusals table, now that the eighth rule exists (issue 1954 D1) --- go/delivery/assigner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/delivery/assigner_test.go b/go/delivery/assigner_test.go index 1adb030..77d7d2a 100644 --- a/go/delivery/assigner_test.go +++ b/go/delivery/assigner_test.go @@ -147,7 +147,7 @@ func TestAssignerValuesRefused(t *testing.T) { {Removed: true, FP: fp2of('a')}, {FP: fp2of('a'), Newest: fp2of('b')}, {FP: fp2of('a'), Newest: fp2of('a'), Rule: 3}, - {FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: 8}, + {FP: fp2of('a'), Composed: hashOf('1'), Newest: fp2of('b'), Rule: 9}, {FP: fp2of('a'), Composed: hashOf('1'), Hold: true}, {FP: fp2of('a'), Composed: hashOf('1'), Put: true, Newest: fp2of('b'), Rule: 3}, {Removed: true, Put: true}, From 66ff33dbd35d487cb073e0ea2d11237fb84d1d7c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:39:29 +0100 Subject: [PATCH 28/52] Name a provision's credential as the vault's own names it, read each node's provisions by its engine, and give mesh-config's held values their layout in the sdk, so the vault, the engine and the composition checks read one form (issue 1954, issue 1953 D7) --- go/buscompose/buscompose_test.go | 11 ++++++++- go/buscompose/engine.go | 6 +++++ go/delivery/control_records.go | 37 +++++++++++++++++++++++++++++ go/delivery/control_records_test.go | 6 +++++ go/provisions/provisions.go | 33 ++++++++++++++++++++++++- go/provisions/provisions_test.go | 10 ++++++++ 6 files changed, 101 insertions(+), 2 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 4e1d0d2..dc82f65 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" "github.com/nats-io/nats.go/jetstream" ) @@ -77,7 +78,9 @@ func engineBeyond(user, g string) bool { strings.HasPrefix(g, "$SRV.") || strings.Contains(g, ".node-engine-"+node) || strings.Contains(g, "KV_"+EngineActsBucket) || strings.HasPrefix(g, "$KV."+EngineActsBucket+"."+node+".") || // And its node's declared packages (novox/hq ADR 0565 clause 6). - strings.Contains(g, "KV_"+EnginePackagesBucket) || g == "$KV."+EnginePackagesBucket+"."+node + strings.Contains(g, "KV_"+EnginePackagesBucket) || g == "$KV."+EnginePackagesBucket+"."+node || + // And the provisions of its node's modules, read (novox/hq ADR 0570 clause 6). + strings.Contains(g, "KV_"+provisions.ProvisionsBucket) } // createsAnAccountsConsumer is the grants part's create and read of a durable consumer an account needs, by its stream @@ -652,6 +655,12 @@ func healthChecks(t *testing.T) { if err != nil { continue } + // ADR 0570 clause 6: a node's engine reads its own node's provision keys, and no other node's, and writes none. + if u.Kind == KindNode && (!MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.a.keycloak.db") || + MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.b.keycloak.db") || + MayPublish(perms, "$KV.mesh-provisioner_provisions.a.keycloak.db")) { + t.Errorf("node %s reads the provisions %v", u.Node, perms.Publish) + } if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_packages.a") || MayPublish(perms, "$KV.mesh-host_packages.b")) { t.Errorf("node %s writes the packages keys %v", u.Node, perms.Publish) } diff --git a/go/buscompose/engine.go b/go/buscompose/engine.go index 2e6a902..4681729 100644 --- a/go/buscompose/engine.go +++ b/go/buscompose/engine.go @@ -4,6 +4,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // The node seat node-engine (novox/hq ADR 0563 §1): every node's engine holds it on its own bus account, and no module @@ -52,6 +53,11 @@ func engineGrants(p Principal) (pub, sub []string) { if own, err := NodeKeysGrants(EngineActsBucket, p.Node); err == nil { pub = append(pub, own...) } + // And the provisions its node's modules consume, its own node's keys alone, read (novox/hq ADR 0570 clause 6: the + // consuming engine checks the vault's name and generation of what it is handed). + if safeSubject.MatchString(p.Node) { + pub = append(pub, readGrants(provisions.ProvisionsBucket, p.Node+".>")...) + } // And its node's declared packages, its own key alone (ADR 0565 clause 6). if safeSubject.MatchString(p.Node) { pub = append(pub, EnginePackagesGrants(p.Node)...) diff --git a/go/delivery/control_records.go b/go/delivery/control_records.go index 9d9f869..a0112d8 100644 --- a/go/delivery/control_records.go +++ b/go/delivery/control_records.go @@ -91,5 +91,42 @@ type DeclaredPackages struct { Packages []PackageDeclared `json:"packages"` } +// mesh-config's state of the approved values held for a pull request (novox/hq ADR 0565 clause 5): read by the checker +// and the assigner's composition check, which lay a pull request's own held values over its modules' values. +const ( + MeshConfigKeptState = "kept" + MeshConfigKeptRead = "mesh-config." + MeshConfigKeptState +) + +// KeptValue is one approved held value, mesh-config's layout unchanged (modules/mesh-config pulls.go): kept apart from +// the values mesh-config resolves, laid only over the composition of its own pull request at its head. +type KeptValue struct { + Pull string `json:"pull"` + Head string `json:"head"` + Digest string `json:"declaration"` + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + // Value is the value held, as the operator approved it. + Value json.RawMessage `json:"value"` + By string `json:"by"` + Via string `json:"via"` + Why string `json:"why"` + // Approved is who approved it, as the proposal kept it. + Approved string `json:"approved"` + Proposal string `json:"proposal"` + At time.Time `json:"at"` +} + +// HeldValueKey is a held value's key in `kept`: the pull request's key (ConfigCandidateKey's form), the node, the module and +// the config key, joined by dots, as mesh-config writes it. +func HeldValueKey(pull, node, module, key string) (string, error) { + p, err := ConfigCandidateKey(pull) + if err != nil { + return "", err + } + return p + "." + node + "." + module + "." + key, nil +} + // The assigner's state of a person's upgrade policy, one key per module (clause 7; catalogue.PersonPolicy). const AssignerPoliciesState = "policies" diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index 6e4d42b..436867f 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -19,6 +19,9 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { `{"package":"nano","modules":["web"],"absent":true}]}`, &DeclaredPackages{}}, {"a config candidate", `{"v":1,"pull":"novox/mesh-catalog#42","head":"abc","declares":{"mail":{"quota":{"kind":"config",` + `"default":10,"why":"a quota","type":"integer","minimum":1}}},"at":"2026-10-29T12:00:00Z"}`, &ConfigCandidate{}}, + {"a held value", `{"pull":"novox/mesh-catalog#42","head":"abc","declaration":"sha256:d","node":"novox","module":"mail",` + + `"key":"quota","value":20,"by":"person.jochen","via":"telegram","why":"more room","approved":"the operator",` + + `"proposal":"p-1","at":"2026-10-29T12:00:00Z"}`, &KeptValue{}}, {"the predefined list", `{"v":1,"commit":"abc","at":"2026-10-29T12:00:00Z","list":{"domain":{"type":"string"}}}`, &PredefinedKept{}}, } { if err := json.Unmarshal([]byte(c.raw), c.into); err != nil { @@ -32,6 +35,9 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { t.Errorf("%s reads back as %s", c.name, again) } } + if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil { + t.Errorf("a held value's key is %q, %v", k, err) + } for pull, want := range map[string]string{"novox/mesh-catalog#42": "novox.mesh-catalog.42", "x#1": "", "a/b": "", "a/b.c#1": ""} { if got, err := ConfigCandidateKey(pull); got != want || (want == "") != (err != nil) { t.Errorf("%s keyed %q, %v", pull, got, err) diff --git a/go/provisions/provisions.go b/go/provisions/provisions.go index 4981dd4..18e1f17 100644 --- a/go/provisions/provisions.go +++ b/go/provisions/provisions.go @@ -92,6 +92,35 @@ func PlanKey(node, module string) (string, error) { return node + "." + module, nil } +// The forms of a provision's credential, as a plan's entry names them (catalogue.PlanProvision.CredentialForm). +const ( + FormPair = "pair" + FormShared = "shared" + FormIssued = "issued" + FormGiven = "given" +) + +// VaultName is the vault's name of a provision's credential in form, as its verb `own` names it and the provider asks +// it: `pair////[/]` for a login of the consumer's own (ADR 0158 +// §11), `provision//` for the provider's one shared value, and `provision////` for a holder's own issued or given value. Empty for a form with no +// credential. +func VaultName(form string, p Provision) string { + switch form { + case FormPair: + n := "pair/" + p.Provider.Node + "/" + p.Provision + "/" + p.Consumer.Node + "/" + p.Consumer.Module + if p.Local != "" { + n += "/" + p.Local + } + return n + case FormShared: + return "provision/" + p.Provider.Node + "/" + p.Provision + case FormIssued, FormGiven: + return "provision/" + p.Provider.Node + "/" + p.Provision + "/" + p.Consumer.Node + "/" + p.Consumer.Module + } + return "" +} + // End is one end of a provision: a module on a node. type End struct { Module string `json:"module"` @@ -99,7 +128,9 @@ type End struct { } // Reference is a value the vault keeps, by its id and the generation of the value: never the value, sealed or not -// (ADR 0349 §7). The holder writes the generation the vault says in its own entry (clause 6). +// (ADR 0349 §7). The holder writes the generation the vault says in its own entry (clause 6). ID is the vault's name of +// the credential exactly as its verb `own` names it (VaultName): the consuming node's engine checks both the name and +// the generation of what it is handed. type Reference struct { ID string `json:"id"` Generation int `json:"generation"` diff --git a/go/provisions/provisions_test.go b/go/provisions/provisions_test.go index 294ef2f..c720f42 100644 --- a/go/provisions/provisions_test.go +++ b/go/provisions/provisions_test.go @@ -72,6 +72,16 @@ func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { if k, err := PlanKey("novox", "restic"); k != "novox.restic" || err != nil { t.Errorf("a plan's key is %q, %v", k, err) } + // A reference's id is the vault's name as its verb own names it, in each form (the host agent, 2026-10-29). + for form, want := range map[string]string{FormPair: "pair/novox/postgres-database/novox/keycloak", FormShared: "provision/novox/postgres-database", + FormIssued: "provision/novox/postgres-database/novox/keycloak", "none": ""} { + if got := VaultName(form, db); got != want { + t.Errorf("the %s name is %q", form, got) + } + } + if got := VaultName(FormPair, Provision{Provision: "p", Local: "r", Provider: End{Node: "a"}, Consumer: End{Node: "b", Module: "m"}}); got != "pair/a/p/b/m/r" { + t.Errorf("a local name's pair is %q", got) + } if ProvisionsBucket != "mesh-provisioner_provisions" || PlansRead != "mesh-provisioner.plans" { t.Errorf("the states are %s and %s", ProvisionsBucket, PlansRead) } From 692dbf9a1791cc7d41dbc117d8d31175d8b78f43 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:50 +0100 Subject: [PATCH 29/52] Bind every provider by mesh-provisioner's plans and provision keys instead of the controller's assignments, act once a provision's dependencies are provided, say the outcome in its entry, and retire a consumer only on the operator's yes (issue 1954 L6-P3, P4, ADR 0570 clauses 2, 3, 5) --- go/provider/consumers_test.go | 3 +- go/provider/ended_test.go | 42 ---- go/provider/harness/grants_moved_test.go | 7 +- go/provider/harness/moved.go | 93 +++----- go/provider/harness/moved_test.go | 136 ++++++------ go/provider/provider.go | 256 ++++++++++++++--------- go/provider/provider_test.go | 219 +++++++++++++++---- go/provider/statebus_test.go | 12 +- 8 files changed, 430 insertions(+), 338 deletions(-) diff --git a/go/provider/consumers_test.go b/go/provider/consumers_test.go index 731fc31..aaf4aa8 100644 --- a/go/provider/consumers_test.go +++ b/go/provider/consumers_test.go @@ -39,8 +39,7 @@ func TestTheConsumersFileAnswersEachConsumersCredential(t *testing.T) { // Wired as the helper's Vault, the file is what makes each consumer's login. bus := newStandIn() - bus.Put(context.Background(), AssignmentsBucket, "ace.letta", []byte(`{"node":"ace","module":"letta", - "provisioning":{"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres"}]}}`)) + assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) var made []string p := New(Options{Node: "novox", Module: "postgres", Bus: bus, Vault: v, Offers: map[string]Offer{"postgres-database": {Serves: map[string]any{"port": 5432}}}, diff --git a/go/provider/ended_test.go b/go/provider/ended_test.go index c4e25cb..45d6f28 100644 --- a/go/provider/ended_test.go +++ b/go/provider/ended_test.go @@ -1,51 +1,9 @@ package provider import ( - "context" - "encoding/json" - "slices" "testing" ) -// Issue 1098: an assignment with `ended` set, or with a `state`, is not a standing assignment: the helper reads it -// as a removed key, so the entry of its consumer is removed and the adapter hears it is gone. The fields the -// controller adds change nothing. -func TestAnEndedAssignmentIsReadAsARemovedKey(t *testing.T) { - bus, m := newStandIn(), &maker{} - v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}} - assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) - p := postgres(bus, v, m) - if _, err := p.Pass(context.Background()); err != nil { - t.Fatal(err) - } - if got := bus.keys(BucketOf("postgres")); !slices.Equal(got, []string{"ace.letta.postgres-database"}) { - t.Fatalf("entries %v", got) - } - for _, mark := range []map[string]any{{"ended": "2026-10-18T01:00:00Z"}, {"state": "pending"}} { - raw := bus.get(AssignmentsBucket, "ace.letta") - var a map[string]any - if err := json.Unmarshal(raw, &a); err != nil { - t.Fatal(err) - } - for k, v := range mark { - a[k] = v - } - raw, _ = json.Marshal(a) - _ = bus.Put(context.Background(), AssignmentsBucket, "ace.letta", raw) - out, err := p.Pass(context.Background()) - if err != nil { - t.Fatal(err) - } - if got := bus.keys(BucketOf("postgres")); len(got) != 0 || len(out) != 1 || out[0].Status != Removed { - t.Fatalf("%v: entries %v, outcomes %+v", mark, got, out) - } - assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) - if _, err := p.Pass(context.Background()); err != nil { - t.Fatal(err) - } - } -} - func TestStandingIsFalseOnlyForAnEndedOrPendingValue(t *testing.T) { for raw, want := range map[string]bool{ `{"node":"a","module":"b"}`: true, diff --git a/go/provider/harness/grants_moved_test.go b/go/provider/harness/grants_moved_test.go index 620aaf8..920bb7a 100644 --- a/go/provider/harness/grants_moved_test.go +++ b/go/provider/harness/grants_moved_test.go @@ -105,7 +105,8 @@ func TestAnUnreadableGrantFileRetiresNothing(t *testing.T) { } } -// A login the grant file stops giving is retired by the usual rules: stable, then retired, never dropped. +// A login the grant file stops giving is not retired by the provider itself (novox/hq ADR 0570 clause 5): every +// retirement waits for the operator's yes on a provision's key. func TestALoginTheGrantFileStopsGivingIsRetired(t *testing.T) { bound, _ := bindings(1) w := newMovedWorld(t, bound) @@ -117,7 +118,7 @@ func TestALoginTheGrantFileStopsGivingIsRetired(t *testing.T) { w.pass() w.granting(map[string]any{"as": "mesh_ace_baserow", "node": "ace", "values": map[string]any{"name": "baserow"}}) w.settled() - if !reflect.DeepEqual(w.a.removed, []string{"mesh_ace_letta"}) || len(w.a.deleted) != 0 { - t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) + if len(w.a.removed) != 0 || len(w.a.deleted) != 0 { + t.Fatalf("retired %v, deleted %v, by itself", w.a.removed, w.a.deleted) } } diff --git a/go/provider/harness/moved.go b/go/provider/harness/moved.go index 392db1a..820431b 100644 --- a/go/provider/harness/moved.go +++ b/go/provider/harness/moved.go @@ -19,13 +19,13 @@ package harness import ( "context" - "encoding/json" "errors" "fmt" "sync" "time" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // MovedSpec is what differs per provider on the moved path. @@ -57,17 +57,17 @@ type Moved struct { // FirstRetry is the first wait before a watch asked too early is asked again (issue 943); a second when zero. FirstRetry time.Duration - mu sync.Mutex // guards assignments and loaded - assignments map[string][]byte - loaded bool - verified map[string]time.Time + mu sync.Mutex // guards plans and loaded + plans map[string][]byte + loaded bool + verified map[string]time.Time } func newMoved(h *Harness, spec MovedSpec) *Moved { h.mu.Lock() h.init() h.mu.Unlock() - return &Moved{h: h, spec: spec, assignments: map[string][]byte{}, verified: map[string]time.Time{}} + return &Moved{h: h, spec: spec, plans: map[string][]byte{}, verified: map[string]time.Time{}} } // Make brings one consumer's resource into being under its login and the vault's credential, as the harness does: @@ -122,77 +122,50 @@ func (m *Moved) Make(ctx context.Context, c provider.Consumer) error { return nil } -// Gone is the helper asking whether it may forget a consumer that no longer binds. **It never retires**: retiring is -// the harness's rule, the same unasked set over StablePasses passes and StableFor, at most RetireAtOnce at a time -// (retirement.go, ADR 0230), run by retire below. So a consumer missing from one read of the plans loses nothing: -// the helper is told no, and asks again at its next pass, until the login is retired, or asked for again. -func (m *Moved) Gone(_ context.Context, e provider.Entry) error { +// Gone is the helper letting a consumer go, which it does only on a retiring provision the operator said yes to (novox/hq +// ADR 0570 clause 5): its login is retired at once, through the adapter, its resource and data kept and marked to +// delete, deleted only by a person's `cleanup delete`. A consumer another plan still binds under the same login loses +// nothing. +func (m *Moved) Gone(ctx context.Context, e provider.Entry) error { h := m.h as := m.spec.Login(e) if as == "" { return nil // refused: nothing was made for it } - // The plans and the grant file as they are now, not as the last retirement pass read them: with either not - // read, nothing is let go. - want, read := m.asked(h.Node) + want, _ := m.asked(h.Node) h.mu.Lock() defer h.mu.Unlock() - asked := want[as] - if m.spec.WithGrants { - _, granted, ok := m.granted(want) - read = read && ok - asked = asked || granted[as] - } - _, held := h.applied[as] - _, retired := h.r.retired[as] - switch { - case !read: - case asked: + if want[as] { return nil // another holder of the same login still binds: nothing to retire - case retired, !held && h.r.seeded: + } + if _, held := h.applied[as]; !held { return nil } - return fmt.Errorf("%s is kept until its retirement is stable (%d passes and %s) and within the bound, or a "+ - "person approves it", as, h.StablePasses, h.StableFor) + if done := h.retire(ctx, []string{as}, "the operator approved its retirement ("+e.Key()+")", nil); len(done) != 1 { + return fmt.Errorf("%s was not retired; it is tried again at the next pass", as) + } + return nil } -// plan is as much of an assignment as is read to learn who binds the provider. -type plan struct { - Node string `json:"node"` - Module string `json:"module"` - Provisioning *struct { - Provisions []struct { - Provision string `json:"provision"` - Provider string `json:"provider"` - ProviderModule string `json:"provider-module"` - // As is who the consumer is at its provider's end, its slug first (issue 788). - As string `json:"as"` - } `json:"provisions"` - } `json:"provisioning"` -} - -// asked is the logins the plans bind the provider for, and whether they could all be read: a plan that is not the -// controller's, or no read yet, is not an answer, so nothing is counted unasked from it (novox/hq issue 241). +// asked is the logins the seat's plans bind the provider for (novox/hq ADR 0570 clause 2), and whether they could all be +// read: a plan that cannot be read, or no read yet, is not an answer (novox/hq issue 241). func (m *Moved) asked(node string) (map[string]bool, bool) { m.mu.Lock() defer m.mu.Unlock() - if !m.loaded || len(m.assignments) == 0 { + if !m.loaded || len(m.plans) == 0 { return nil, false } want := map[string]bool{} - for key, raw := range m.assignments { - var a plan - if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key { + for key, raw := range m.plans { + pl, err := provisions.DecodePlan(raw) + if err != nil || pl.Node+"."+pl.Module != key { return nil, false } - if a.Provisioning == nil { - continue - } - for _, b := range a.Provisioning.Provisions { + for _, b := range pl.Plan.Provisions { if b.Provider == node && b.ProviderModule == m.spec.Module && b.Provision == m.spec.Provision { as := b.As if as == "" { - as = provider.ConsumerIdentity(a.Node, a.Module) // a plan that names none + as = provider.ConsumerIdentity(pl.Node, pl.Module) // a plan that names none } want[as] = true } @@ -223,8 +196,8 @@ func (m *Moved) granted(planned map[string]bool) ([]contribution, map[string]boo return own, logins, true } -// retire is one pass of the harness's retirement over what the plans bind and the grant file gives now; the logins -// only the grant file gives are made and checked first, as the grant path does. +// retire is one pass over what the plans bind and the grant file gives now: the logins only the grant file gives are +// made and checked, as the grant path does. It retires nothing (ADR 0570 clause 5). func (m *Moved) retire(ctx context.Context) { h := m.h want, ok := m.asked(h.Node) @@ -245,7 +218,7 @@ func (m *Moved) retire(ctx context.Context) { } h.seed(ctx, want) h.apply(ctx, given) - h.retireUnasked(ctx, want) + // No login is retired for going unasked (novox/hq ADR 0570 clause 5): only Gone, on the operator's yes. h.r.lastWant = want for as := range h.failing { if !want[as] { @@ -261,13 +234,13 @@ func (m *Moved) retire(ctx context.Context) { // watch keeps the plans as the bus holds them, from the first read on. func (m *Moved) watch(ctx context.Context, bus provider.Bus) error { - err := bus.Watch(ctx, provider.AssignmentsBucket, func(key string, value []byte) { + err := bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) { m.mu.Lock() defer m.mu.Unlock() if value == nil { - delete(m.assignments, key) + delete(m.plans, key) } else { - m.assignments[key] = append([]byte(nil), value...) + m.plans[key] = append([]byte(nil), value...) } }) if err == nil { diff --git a/go/provider/harness/moved_test.go b/go/provider/harness/moved_test.go index ce79065..bce61ff 100644 --- a/go/provider/harness/moved_test.go +++ b/go/provider/harness/moved_test.go @@ -11,7 +11,9 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" ) const ( @@ -57,6 +59,43 @@ func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { return nil } +// putPlan is a consumer's plan on the seat mesh-provisioner, of these plan entries, each provision's key provided (novox/hq +// ADR 0570 clauses 1, 2): what the helper binds by. +func (b *bucketBus) putPlan(node, module string, entries ...map[string]any) { + raw, _ := json.Marshal(entries) + var ps []catalogue.PlanProvision + if err := json.Unmarshal(raw, &ps); err != nil { + panic(err) + } + for _, pp := range ps { + pr := provisions.Provision{Provision: pp.Provision, Local: pp.Local, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: pp.ProviderModule, Node: pp.Provider}, State: provisions.Provided} + key, _ := pr.Key() + value, err := provisions.Encode(pr) + if err != nil { + panic(err) + } + _ = b.Put(context.Background(), provisions.ProvisionsBucket, key, value) + } + plan, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: catalogue.Plan{Provisions: ps}}) + if err != nil { + panic(err) + } + _ = b.Put(context.Background(), provisions.PlansBucket, node+"."+module, plan) +} + +// retireOn is a consumer's provision retiring with the operator's yes (ADR 0570 clause 5), its plan gone: the one way +// a provider lets it go. +func (b *bucketBus) retireOn(node, module, provides string) { + pr := provisions.Provision{Provision: ProvisionName, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: provides, Node: "novox"}, State: provisions.Retiring, Why: module + " left", + Warrant: &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}} + key, _ := pr.Key() + value, _ := provisions.Encode(pr) + _ = b.Delete(context.Background(), provisions.PlansBucket, node+"."+module) + _ = b.Put(context.Background(), provisions.ProvisionsBucket, key, value) +} + // bind is one consumer's assignment, whose plan names this machine's postgres as the provider and carries what the // consumer contributes (issue 776). func (b *bucketBus) bind(node, module string, contributes map[string]any) { @@ -65,12 +104,8 @@ func (b *bucketBus) bind(node, module string, contributes map[string]any) { // bindNamed is bind to a provider module, with the names the controller composed for the consumer (issue 792). func (b *bucketBus) bindNamed(provides, node, module string, contributes map[string]any, names map[string]string) { - plan, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": provides, - "contributes": contributes, "names": names}}}}) - if err := b.Put(context.Background(), provider.AssignmentsBucket, node+"."+module, plan); err != nil { - panic(err) - } + b.putPlan(node, module, map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": provides, + "contributes": contributes, "names": names}) } // consumersFile writes the file the machine's client fills: one credential per consumer. @@ -124,9 +159,9 @@ func newMovedWorldOf(t *testing.T, bound map[[2]string]string, spec MovedSpec) * func (w *movedWorld) pass() []provider.Outcome { w.now = w.now.Add(5 * time.Second) // the plans as the bus holds them now, as the watch would hand them - w.run.assignments = map[string][]byte{} - for k, v := range w.bus.buckets[provider.AssignmentsBucket] { - w.run.assignments[k] = v + w.run.plans = map[string][]byte{} + for k, v := range w.bus.buckets[provisions.PlansBucket] { + w.run.plans[k] = v } // the helper reads its buckets from the in-memory bus w.p = provider.New(provider.Options{Node: "novox", Module: w.spec.Module, Bus: w.bus, @@ -205,46 +240,27 @@ func TestTheConsumersFileMakesWhatTheGrantFilesMake(t *testing.T) { } } -// One read without a consumer retires nothing: it takes the harness's stable passes and time, and then the entry -// goes and the login is retired, never dropped. +// A consumer whose plan goes retires nothing by itself, however long (novox/hq ADR 0570 clause 5): its login is retired, +// never dropped, and its entry removed only once its provision is retiring with the operator's yes. func TestAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { bound, logins := bindings(2) w := newMovedWorld(t, bound) w.bus.bind("ace", "app1", nil) w.bus.bind("ace", "app2", nil) w.pass() - - // One read without app2. - saved := w.bus.buckets[provider.AssignmentsBucket]["ace.app2"] - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") - w.pass() - if len(w.a.removed) != 0 { - t.Fatalf("retired %v on one read", w.a.removed) - } - if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; !kept { - t.Fatal("the entry went on one read") - } - // It is back: nothing was lost, and the entry never left the bucket. - w.bus.Put(ctx, provider.AssignmentsBucket, "ace.app2", saved) + _ = w.bus.Delete(ctx, provisions.PlansBucket, "ace.app2") for i := 0; i < 8; i++ { w.pass() } - w.now = w.now.Add(15 * time.Minute) + w.now = w.now.Add(30 * time.Minute) w.pass() - if len(w.a.removed) != 0 || len(w.bus.buckets[provider.BucketOf("postgres")]) != 2 { - t.Fatalf("retired %v, entries %d", w.a.removed, len(w.bus.buckets[provider.BucketOf("postgres")])) + if len(w.a.removed) != 0 || w.h.r.waiting != nil { + t.Fatalf("retired %v, waiting %v, with no warrant", w.a.removed, w.h.r.waiting) } - - // Gone for good: not retired before StablePasses and StableFor, then retired and its entry removed. - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") - for i := 0; i < 6; i++ { - w.pass() + if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; !kept { + t.Fatal("the entry went with no warrant") } - if len(w.a.removed) != 0 { - t.Fatalf("retired %v before the time was out", w.a.removed) - } - w.now = w.now.Add(11 * time.Minute) - w.pass() + w.bus.retireOn("ace", "app2", "postgres") w.pass() if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 { t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) @@ -254,35 +270,13 @@ func TestAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { } } -// More than three at once waits for a person, as the harness's bound says. -func TestMoreThanTheBoundWaitsForAPerson(t *testing.T) { - bound, _ := bindings(5) - w := newMovedWorld(t, bound) - for i := 1; i <= 5; i++ { - w.bus.bind("ace", fmt.Sprintf("app%d", i), nil) - } - w.pass() - for i := 1; i <= 4; i++ { - w.bus.Delete(ctx, provider.AssignmentsBucket, fmt.Sprintf("ace.app%d", i)) - } - for i := 0; i < 6; i++ { - w.pass() - } - w.now = w.now.Add(11 * time.Minute) - w.pass() - w.pass() - if len(w.a.removed) != 0 || w.h.r.waiting == nil { - t.Fatalf("retired %v, waiting %v", w.a.removed, w.h.r.waiting) - } -} - // A plan list that was never read is not nobody asking. func TestNoPlansReadRetiresNothing(t *testing.T) { bound, _ := bindings(1) w := newMovedWorld(t, bound) w.bus.bind("ace", "app1", nil) w.pass() - w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app1") + w.bus.Delete(ctx, provisions.PlansBucket, "ace.app1") for i := 0; i < 8; i++ { w.pass() } @@ -310,12 +304,8 @@ func TestAConsumerWithoutACredentialIsWaitedFor(t *testing.T) { func TestASluggedConsumerKeepsTheLoginItsPlanNames(t *testing.T) { agent := provider.ConsumerIdentity("ace", "agent") w := newMovedWorld(t, map[[2]string]string{{"ace", "claude-code"}: "pw-agent"}) - plan, _ := json.Marshal(map[string]any{"node": "ace", "module": "claude-code", "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", - "provider-module": "postgres", "as": agent}}}}) - if err := w.bus.Put(ctx, provider.AssignmentsBucket, "ace.claude-code", plan); err != nil { - t.Fatal(err) - } + w.bus.putPlan("ace", "claude-code", map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": "postgres", "as": agent}) for _, o := range w.pass() { if o.Status != provider.Answered { t.Fatalf("%+v", o) @@ -353,10 +343,8 @@ func TestASpecWithoutGrantsMakesUnderThePlansAsAndIgnoresTheGrantFile(t *testing if err != nil { t.Fatal(err) } - plan, _ := json.Marshal(map[string]any{"node": "ace", "module": "wiki", "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", - "provider-module": "keycloak", "as": "wiki-client", "contributes": map[string]any{"label": "Wiki"}}}}}) - _ = bus.Put(ctx, provider.AssignmentsBucket, "ace.wiki", plan) + bus.putPlan("ace", "wiki", map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": "keycloak", "as": "wiki-client", "contributes": map[string]any{"label": "Wiki"}}) if err := run.watch(ctx, bus); err != nil { t.Fatal(err) } @@ -454,12 +442,8 @@ func TestTheConsumersFileMakesWhatTheGrantFileMakesForTheKeycloakShape(t *testin func TestAConsumerNamedByItsPlanKeepsItsClientId(t *testing.T) { agent := provider.ConsumerIdentity("home", "dashboard") w := newMovedWorldOf(t, map[[2]string]string{{"home", "grafana-2"}: "pw"}, keycloakSpec) - plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana-2", "provisioning": map[string]any{ - "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", - "as": agent, "contributes": grafanaContributes, "names": grafanaNames}}}}) - if err := w.bus.Put(ctx, provider.AssignmentsBucket, "home.grafana-2", plan); err != nil { - t.Fatal(err) - } + w.bus.putPlan("home", "grafana-2", map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", + "as": agent, "contributes": grafanaContributes, "names": grafanaNames}) for _, o := range w.pass() { if o.Status != provider.Answered { t.Fatalf("%+v", o) diff --git a/go/provider/provider.go b/go/provider/provider.go index 4adee9e..c183514 100644 --- a/go/provider/provider.go +++ b/go/provider/provider.go @@ -47,6 +47,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // AssignmentsBucket is the controller's assignments mirror, `.`, whose value carries the plan of a @@ -124,6 +125,11 @@ type Entry struct { VaultName string `json:"vault-name,omitempty"` // Refused, when set, is why the consumer is not served, naming both ends. Refused string `json:"refused,omitempty"` + // State is the provider's outcome for the consumer, which the holder of mesh-provisioner reads to set the + // provision's key (novox/hq ADR 0570 clause 3): provided, failed with Why, or waits with what is awaited. Empty from + // a provider before it. + State string `json:"state,omitempty"` + Why string `json:"why,omitempty"` // Contributes is what the consumer's module tells its provider about this provision (a database's name, its // extensions), from its plan's entry (issue 776). Handed to Make and Gone; never written to the bucket. Contributes map[string]any `json:"-"` @@ -185,11 +191,13 @@ type Options struct { type Provider struct { opts Options - mu sync.Mutex - assignments map[string][]byte - entries map[string][]byte - said map[string]string - poke chan struct{} + mu sync.Mutex + entries map[string][]byte + // plans and provisions are the seat mesh-provisioner's states (ADR 0570 clauses 1, 2), by key. + plans map[string][]byte + provisions map[string][]byte + said map[string]string + poke chan struct{} } // New is a provider answering its consumers as opts say. @@ -203,8 +211,8 @@ func New(opts Options) *Provider { if opts.Logf == nil { opts.Logf = func(f string, a ...any) { fmt.Printf("[provider] "+f+"\n", a...) } } - return &Provider{opts: opts, assignments: map[string][]byte{}, entries: map[string][]byte{}, - said: map[string]string{}, poke: make(chan struct{}, 1)} + return &Provider{opts: opts, entries: map[string][]byte{}, + plans: map[string][]byte{}, provisions: map[string][]byte{}, said: map[string]string{}, poke: make(chan struct{}, 1)} } // What a pass did for one bind. @@ -232,16 +240,22 @@ func (p *Provider) Run(ctx context.Context) error { if err := p.watch(ctx, BucketOf(p.opts.Module), p.entries); err != nil { return fmt.Errorf("watching %s: %w", BucketOf(p.opts.Module), err) } - if err := p.watch(ctx, AssignmentsBucket, p.assignments); err != nil { - return fmt.Errorf("watching %s: %w", AssignmentsBucket, err) + for _, bucket := range []string{provisions.PlansBucket, provisions.ProvisionsBucket} { + into := p.plans + if bucket == provisions.ProvisionsBucket { + into = p.provisions + } + if err := p.watch(ctx, bucket, into); err != nil { + return fmt.Errorf("watching %s: %w", bucket, err) + } } tick := time.NewTicker(p.opts.Every) defer tick.Stop() for { p.mu.Lock() - assignments, entries := clone(p.assignments), clone(p.entries) + in := Inputs{Entries: clone(p.entries), Plans: clone(p.plans), Provisions: clone(p.provisions)} p.mu.Unlock() - if _, err := p.pass(ctx, assignments, entries); err != nil { + if _, err := p.passIn(ctx, in); err != nil { p.say("pass", err.Error()) } select { @@ -262,7 +276,7 @@ func (p *Provider) watch(ctx context.Context, bucket string, into map[string][]b into[key] = append([]byte(nil), value...) } p.mu.Unlock() - if bucket == AssignmentsBucket { + if bucket != BucketOf(p.opts.Module) { select { case p.poke <- struct{}{}: default: @@ -302,110 +316,82 @@ func (p *Provider) Pass(ctx context.Context) ([]Outcome, error) { if err != nil { return nil, fmt.Errorf("reading %s: %w", BucketOf(p.opts.Module), err) } - assignments, err := read(AssignmentsBucket) + plans, err := read(provisions.PlansBucket) if err != nil { - return nil, fmt.Errorf("reading %s: %w", AssignmentsBucket, err) + return nil, fmt.Errorf("reading %s: %w", provisions.PlansBucket, err) } - return p.pass(ctx, assignments, entries) + provs, err := read(provisions.ProvisionsBucket) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", provisions.ProvisionsBucket, err) + } + return p.passIn(ctx, Inputs{Entries: entries, Plans: plans, Provisions: provs}) } -// planned is as much of an assignment as the helper reads: lenient, since the plan is the client's to act on. -type planned struct { - Node string `json:"node"` - Module string `json:"module"` - Provisioning *struct { - Provisions []struct { - Provision string `json:"provision"` - Local string `json:"local"` - Provider string `json:"provider"` - ProviderModule string `json:"provider-module"` - Contributes map[string]any `json:"contributes"` - At string `json:"at"` - From string `json:"from"` - As string `json:"as"` - Names map[string]string `json:"names"` - CredentialForm string `json:"credential-form"` - Name string `json:"name"` - } `json:"provisions"` - } `json:"provisioning"` +// Inputs are what one pass reads: the provider's own entries, and the seat mesh-provisioner's plans and provisions +// (novox/hq ADR 0570 clauses 1, 2), each by key. The controller's assignments are read no more (clause 2). +type Inputs struct { + Entries, Plans, Provisions map[string][]byte } -func (p *Provider) pass(ctx context.Context, assignments, entries map[string][]byte) ([]Outcome, error) { +func (p *Provider) passIn(ctx context.Context, in Inputs) ([]Outcome, error) { + entries := in.Entries var binds []Entry var out []Outcome unknown := map[string]bool{} // consumers whose assignment was skipped: their entries are not judged - for _, key := range sortedKeys(assignments) { - if !Standing(assignments[key]) { - continue // ended or pending: read as a removed key (issue 1098) - } - var a planned - if err := json.Unmarshal(assignments[key], &a); err != nil || a.Node+"."+a.Module != key { - o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the assignment %s is not the controller's, so "+ - "who it binds is not known; it is skipped, and its entries are left as they are", key)} + // Every consumer is bound by its plan and its provision keys (ADR 0570 clauses 2, 3), and an entry no plan names is + // never let go by the provider itself (clause 5): only a retiring key with the operator's yes retires it. + seat := map[string]bool{} + gated := map[string]provisions.Provision{} + for _, key := range sortedKeys(in.Plans) { + plan, err := provisions.DecodePlan(in.Plans[key]) + if err != nil || plan.Node+"."+plan.Module != key { + o := Outcome{Key: key, Status: Failed, Why: fmt.Sprintf("the plan %s is not readable, so who it binds is not "+ + "known; it is skipped, and its entries are left as they are", key)} out = append(out, o) p.say(key, o.Status+" "+o.Why) - unknown[key] = true + unknown[key], seat[key] = true, true continue } - if a.Provisioning == nil { - continue - } - for _, b := range a.Provisioning.Provisions { + seat[key] = true + for _, b := range plan.Plan.Provisions { if b.Provider != p.opts.Node || b.ProviderModule != p.opts.Module { continue } - binds = append(binds, Entry{Provider: p.opts.Node, ProviderModule: p.opts.Module, - Provision: b.Provision, Consumer: a.Node, Module: a.Module, Local: b.Local, - Contributes: b.Contributes, At: b.At, From: b.From, As: b.As, Names: b.Names, - CredentialForm: b.CredentialForm, Name: b.Name}) + e := Entry{Provider: p.opts.Node, ProviderModule: p.opts.Module, Provision: b.Provision, Consumer: plan.Node, + Module: plan.Module, Local: b.Local, Contributes: b.Contributes, At: b.At, From: b.From, As: b.As, + Names: b.Names, CredentialForm: b.CredentialForm, Name: b.Name} + binds = append(binds, e) + } + } + for _, key := range sortedKeys(in.Provisions) { + if pr, err := provisions.Decode(in.Provisions[key]); err == nil && pr.Provider.Node == p.opts.Node && + pr.Provider.Module == p.opts.Module { + gated[key] = pr } } - want := map[string]bool{} for _, b := range binds { key := b.Key() want[key] = true - o := p.answer(ctx, b, entries[key]) - out = append(out, o) - p.say(key, o.Status+" "+o.Why) - } - for _, key := range sortedKeys(entries) { - if want[key] || unknown[consumerOf(key)] { - continue - } - var e Entry - if err := json.Unmarshal(entries[key], &e); err != nil || e.Provider != p.opts.Node || - e.ProviderModule != p.opts.Module { - continue // another machine's instance of this provider wrote it, or nothing of this helper's - } - if p.opts.Gone != nil { - if err := p.opts.Gone(ctx, e); err != nil { - o := Outcome{Key: key, Status: Failed, Why: "no longer bound, and the provider could not let it go: " + - err.Error()} - out = append(out, o) - p.say(key, o.Status+" "+o.Why) - continue - } - } - if err := p.remove(ctx, key, entries[key]); err != nil { - if errors.Is(err, stdio.ErrChanged) { - o := Outcome{Key: key, Status: Superseded, Why: "another instance of the provider wrote its entry " + - "since this pass read it, so it stands"} - out = append(out, o) - p.say(key, o.Status+" "+o.Why) - continue - } - o := Outcome{Key: key, Status: Failed, Why: "no longer bound, and its entry could not be removed: " + - err.Error()} + if why, ok := p.ready(key, gated, in.Provisions); !ok { + o := Outcome{Key: key, Status: Waiting, Why: why} out = append(out, o) p.say(key, o.Status+" "+o.Why) continue } - out = append(out, Outcome{Key: key, Status: Removed}) - p.say(key, Removed) - p.mu.Lock() - delete(p.said, key) - p.mu.Unlock() + o := p.answer(ctx, b, entries[key]) + out = append(out, o) + p.say(key, o.Status+" "+o.Why) + } + // ADR 0570 clause 5: a retiring key the operator said yes to is let go; one asked and not answered, or declined, + // keeps its entry. + for _, key := range sortedKeys(gated) { + if pr := gated[key]; pr.State == provisions.Retiring && pr.Warrant != nil && pr.Warrant.Option == "yes" { + want[key] = false + if _, held := entries[key]; held { + out = append(out, p.letGo(ctx, key, entries[key])...) + } + } } return out, nil } @@ -443,6 +429,7 @@ func (p *Provider) answer(ctx context.Context, e Entry, have []byte) Outcome { strings.Join(derivedKeys(offer.Serves), ", ")) } if e.Refused != "" { + e.State, e.Why = provisions.Failed, e.Refused return p.write(ctx, e, have, Outcome{Key: key, Status: Refused, Why: e.Refused}) } @@ -455,6 +442,7 @@ func (p *Provider) answer(ctx context.Context, e Entry, have []byte) Outcome { e.Refused = fmt.Sprintf("%s on %s serving %s to %s on %s: %v", e.ProviderModule, e.Provider, e.Provision, e.Module, e.Consumer, err) e.As = "" + e.State, e.Why = provisions.Failed, e.Refused return p.write(ctx, e, have, Outcome{Key: key, Status: Refused, Why: e.Refused}) } if e.At != "" || e.From != "" { @@ -506,11 +494,15 @@ func (p *Provider) answer(ctx context.Context, e Entry, have []byte) Outcome { } if p.opts.Make != nil { if err := p.opts.Make(ctx, Consumer{Entry: e, Credential: credential}); err != nil { - why := err.Error() - why = redact(why, credential) - return Outcome{Key: key, Status: Failed, Why: "the provider could not make it: " + why} + why := "the provider could not make it: " + redact(err.Error(), credential) + // Said in its entry, which the holder of mesh-provisioner reads (ADR 0570 clause 3), with nothing served. + failed := e + failed.Values, failed.VaultName, failed.User, failed.State, failed.Why = nil, "", "", provisions.Failed, why + p.write(ctx, failed, have, Outcome{Key: key, Status: Failed, Why: why}) + return Outcome{Key: key, Status: Failed, Why: why} } } + e.State = provisions.Provided return p.write(ctx, e, have, Outcome{Key: key, Status: Answered}) } @@ -557,20 +549,20 @@ func (p *Provider) say(key, what string) { } // SharedName is the vault's one value per provider and provision (ADR 0158, issue 677). -func SharedName(provider, provision string) string { return "provision/" + provider + "/" + provision } +func SharedName(provider, provision string) string { + return provisions.VaultName(provisions.FormShared, provisions.Provision{Provision: provision, Provider: provisions.End{Node: provider}}) +} // HolderName is the vault's value of one holder's own credential, issued or given for it (mesh-vault perholder.go). func HolderName(provider, provision, node, module string) string { - return "provision/" + provider + "/" + provision + "/" + node + "/" + module + return provisions.VaultName(provisions.FormIssued, provisions.Provision{Provision: provision, + Provider: provisions.End{Node: provider}, Consumer: provisions.End{Node: node, Module: module}}) } // PairName is the vault's value of one consumer's own login (ADR 0048, ADR 0158 §11, issue 677's note 12). func PairName(provider, provision, node, module, local string) string { - n := "pair/" + provider + "/" + provision + "/" + node + "/" + module - if local != "" { - n += "/" + local - } - return n + return provisions.VaultName(provisions.FormPair, provisions.Provision{Provision: provision, Local: local, + Provider: provisions.End{Node: provider}, Consumer: provisions.End{Node: node, Module: module}}) } // identityUnusable is every character a consumer's identity does not hold (mesh-controller internal/catalogue @@ -672,8 +664,9 @@ func sortedKeys[V any](m map[string]V) []string { } // StateBus is the Bus of a provider that runs as a module bundle over the runtime's stdio: the provider's own bucket, -// BucketOf(module), is the module's state `bound`, and AssignmentsBucket is the state it reads, -// `mesh-controller.assignments` (both declared in the module's manifest). Any other bucket is refused. +// BucketOf(module), is the module's state `bound`, and provisions.PlansBucket and provisions.ProvisionsBucket are the +// states it reads, `mesh-provisioner.plans` and `mesh-provisioner.provisions` (each declared in the module's manifest). +// Any other bucket is refused. // // A stdio state watch lasts as long as the process and cannot be ended (stdio.KeptState.Watch), so Watch honours the // Bus contract by silencing instead: once ctx has ended the handler is never called again, though the runtime keeps @@ -687,8 +680,10 @@ func (b stateBus) state(bucket string) (stdio.KeptState, error) { switch bucket { case BucketOf(b.module): return stdio.State("bound"), nil - case AssignmentsBucket: - return stdio.State("mesh-controller.assignments"), nil + case provisions.PlansBucket: + return stdio.State(provisions.PlansRead), nil + case provisions.ProvisionsBucket: + return stdio.State(provisions.ProvisionsRead), nil } return stdio.KeptState{}, fmt.Errorf("%s is not a bucket this provider keeps or reads", bucket) } @@ -767,3 +762,56 @@ func fieldSet(f json.RawMessage) bool { s := strings.TrimSpace(string(f)) return s != "" && s != "null" && s != `""` } + +// letGo lets a consumer's entry go: the provider hears it gone (Gone, which retires what it kept for the consumer), and +// the entry is removed while it is still as read. +func (p *Provider) letGo(ctx context.Context, key string, held []byte) []Outcome { + var e Entry + if err := json.Unmarshal(held, &e); err != nil || e.Provider != p.opts.Node || e.ProviderModule != p.opts.Module { + return nil // another machine's instance of this provider wrote it, or nothing of this helper's + } + said := func(o Outcome) []Outcome { + p.say(key, o.Status+" "+o.Why) + return []Outcome{o} + } + if p.opts.Gone != nil { + if err := p.opts.Gone(ctx, e); err != nil { + return said(Outcome{Key: key, Status: Failed, Why: "no longer bound, and the provider could not let it go: " + err.Error()}) + } + } + if err := p.remove(ctx, key, held); err != nil { + if errors.Is(err, stdio.ErrChanged) { + return said(Outcome{Key: key, Status: Superseded, Why: "another instance of the provider wrote its entry " + + "since this pass read it, so it stands"}) + } + return said(Outcome{Key: key, Status: Failed, Why: "no longer bound, and its entry could not be removed: " + err.Error()}) + } + p.say(key, Removed) + p.mu.Lock() + delete(p.said, key) + p.mu.Unlock() + return []Outcome{{Key: key, Status: Removed}} +} + +// ready says whether the provision under key may be made now (novox/hq ADR 0570 clause 3): its key is on the seat, it is +// asked or provided, and every provision it depends on is provided. Otherwise why it waits, naming the key or the +// dependency. +func (p *Provider) ready(key string, mine map[string]provisions.Provision, all map[string][]byte) (string, bool) { + pr, ok := mine[key] + if !ok { + return "mesh-provisioner holds no key " + key + " naming this provider yet", false + } + if pr.State != provisions.Asked && pr.State != provisions.Provided { + return "its key is " + pr.State + ": " + pr.Why, false + } + for _, d := range pr.DependsOn { + dep, err := provisions.Decode(all[d]) + if err != nil { + return "it depends on " + d + ", which mesh-provisioner holds no readable key of", false + } + if dep.State != provisions.Provided { + return "it depends on " + d + ", which is " + dep.State, false + } + } + return "", true +} diff --git a/go/provider/provider_test.go b/go/provider/provider_test.go index 659e35f..806bb34 100644 --- a/go/provider/provider_test.go +++ b/go/provider/provider_test.go @@ -15,6 +15,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // standIn is a stand-in bus: buckets of keys and values, and watches that hand the current values first and then @@ -165,22 +167,63 @@ func (m *maker) Gone(_ context.Context, e Entry) error { return nil } -// assign puts a consumer's assignment with its plan's provisions into the stand-in mirror. -func assign(t *testing.T, bus *standIn, node, module string, provisions ...map[string]string) { +// assign puts a consumer's plan with its provisions on the seat mesh-provisioner, each provision's key provided, as its +// holder writes them (novox/hq ADR 0570 clauses 1, 2): what a provider binds by. +func assign(t *testing.T, bus *standIn, node, module string, binds ...map[string]string) { t.Helper() - ps := []map[string]string{} - ps = append(ps, provisions...) - raw, err := json.Marshal(map[string]any{"node": node, "module": module, "assigned": "2026-10-15T00:00:00Z", - "provisioning": map[string]any{"templates": map[string]string{"source": "x", "digest": "y"}, - "provisions": ps}}) + var ps []catalogue.PlanProvision + for _, b := range binds { + raw, _ := json.Marshal(b) + var pp catalogue.PlanProvision + if err := json.Unmarshal(raw, &pp); err != nil { + t.Fatal(err) + } + ps = append(ps, pp) + } + planned(t, bus, node, module, ps...) +} + +// planned puts a consumer's plan of these provisions on the seat, each provision's key provided. +func planned(t *testing.T, bus *standIn, node, module string, ps ...catalogue.PlanProvision) { + t.Helper() + for _, pp := range ps { + pr := provisions.Provision{Provision: pp.Provision, Local: pp.Local, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: pp.ProviderModule, Node: pp.Provider}, State: provisions.Provided} + key, err := pr.Key() + if err != nil { + t.Fatal(err) + } + value, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Put(context.Background(), provisions.ProvisionsBucket, key, value) + } + raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: catalogue.Plan{Provisions: ps}}) if err != nil { t.Fatal(err) } - if err := bus.Put(context.Background(), AssignmentsBucket, node+"."+module, raw); err != nil { + if err := bus.Put(context.Background(), provisions.PlansBucket, node+"."+module, raw); err != nil { t.Fatal(err) } } +// retire marks a consumer's provision retiring, with the operator's yes (novox/hq ADR 0570 clause 5): the one way a +// provider lets a consumer go. +func retire(t *testing.T, bus *standIn, node, module, provision, provider, providerModule string) { + t.Helper() + pr := provisions.Provision{Provision: provision, Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: providerModule, Node: provider}, State: provisions.Retiring, Why: module + " left " + node, + Warrant: &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}} + key, _ := pr.Key() + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Delete(context.Background(), provisions.PlansBucket, node+"."+module) + _ = bus.Put(context.Background(), provisions.ProvisionsBucket, key, raw) +} + func bind(provision, local, provider, providerModule string) map[string]string { b := map[string]string{"provision": provision, "provider": provider, "provider-module": providerModule} if local != "" { @@ -352,7 +395,7 @@ func TestAnUnbindRemovesTheEntryAndLeavesAnotherProvidersAlone(t *testing.T) { if _, err := p.Pass(context.Background()); err != nil { t.Fatal(err) } - _ = bus.Delete(context.Background(), AssignmentsBucket, "ace.letta") + retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres") out, err := p.Pass(context.Background()) if err != nil { t.Fatal(err) @@ -418,11 +461,11 @@ func TestOnlyBindsNamingThisProviderAreAnswered(t *testing.T) { } } -// An assignment that cannot be read is skipped and said, by its key and why; the rest of the pass goes on (issue 825, +// A plan that cannot be read is skipped and said, by its key and why; the rest of the pass goes on (issue 825, // finding 3). Its consumer's own entry is left alone, since a bind missed would read as an unbind. func TestAnUnreadableAssignmentIsSkippedAndSaidAndTheRestGoesOn(t *testing.T) { for name, bad := range map[string]string{"not json": "{not json", - "another key": `{"node":"g14","module":"umami"}`} { + "another key": `{"v":1,"node":"g14","module":"umami","plan":{"templates":{"source":"","digest":""},"files":null}}`} { bus, m := newStandIn(), &maker{} v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s", "pair/novox/postgres-database/g14/umami": "u"}} @@ -431,7 +474,7 @@ func TestAnUnreadableAssignmentIsSkippedAndSaidAndTheRestGoesOn(t *testing.T) { if _, err := p.Pass(context.Background()); err != nil { t.Fatal(err) } - _ = bus.Put(context.Background(), AssignmentsBucket, "ace.letta", []byte(bad)) + _ = bus.Put(context.Background(), provisions.PlansBucket, "ace.letta", []byte(bad)) assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) var logged []string p.opts.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } @@ -457,7 +500,8 @@ func TestAnUnreadableAssignmentIsSkippedAndSaidAndTheRestGoesOn(t *testing.T) { } } -// An adapter that fails is said with the credential taken out of its words, and the entry is not written. +// An adapter that fails is said with the credential taken out of its words, and its entry says it failed, serving +// nothing (novox/hq ADR 0570 clause 3: the provider's outcome is its entry, which the holder of mesh-provisioner reads). func TestAFailedMakeIsSaidWithoutTheCredential(t *testing.T) { bus, m := newStandIn(), &maker{fail: errors.New(`password "hunter2-secret" rejected`)} v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "hunter2-secret"}} @@ -475,8 +519,11 @@ func TestAFailedMakeIsSaidWithoutTheCredential(t *testing.T) { if strings.Contains(strings.Join(logged, "\n"), "hunter2") || len(logged) == 0 { t.Fatalf("logged %q", logged) } - if bus.keys(BucketOf("postgres")) != nil { - t.Fatal("an entry was written for a consumer the adapter could not make") + var e Entry + raw := bus.get(BucketOf("postgres"), "ace.letta.postgres-database") + if err := json.Unmarshal(raw, &e); err != nil || e.State != "failed" || e.Values != nil || e.VaultName != "" || + strings.Contains(string(raw), "hunter2") { + t.Fatalf("the failed entry is %s", raw) } } @@ -492,7 +539,7 @@ func TestRunAnswersBindsAndUnbindsAsTheyAreWatched(t *testing.T) { go func() { done <- p.Run(ctx) }() assign(t, bus, "ace", "letta", bind("postgres-database", "", "novox", "postgres")) waitFor(t, func() bool { return bus.get(BucketOf("postgres"), "ace.letta.postgres-database") != nil }) - _ = bus.Delete(context.Background(), AssignmentsBucket, "ace.letta") + retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres") waitFor(t, func() bool { return bus.get(BucketOf("postgres"), "ace.letta.postgres-database") == nil }) cancel() if err := <-done; err != nil && !errors.Is(err, context.Canceled) { @@ -532,12 +579,12 @@ func TestAConsumersContributionIsHandedToTheProviderPerConsumer(t *testing.T) { bus, m := newStandIn(), &maker{} v := &vault{values: map[string]string{ "pair/novox/postgres-database/ace/nextcloud": "a", "pair/novox/postgres-database/g14/umami": "b"}} - raw := `{"node":"ace","module":"nextcloud","provisioning":{"templates":{"source":"x","digest":"y"},` + - `"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres",` + - `"contributes":{"name":"nextcloud","extensions":["pg_trgm"]}}]}}` - if err := bus.Put(context.Background(), AssignmentsBucket, "ace.nextcloud", []byte(raw)); err != nil { + var pp catalogue.PlanProvision + if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+ + `"contributes":{"name":"nextcloud","extensions":["pg_trgm"]}}`), &pp); err != nil { t.Fatal(err) } + planned(t, bus, "ace", "nextcloud", pp) assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil { t.Fatal(err) @@ -561,12 +608,12 @@ func TestAConsumersEntryCarriesAtFromAndTheAsOfItsPlan(t *testing.T) { bus, m := newStandIn(), &maker{} v := &vault{values: map[string]string{ "pair/novox/postgres-database/ace/claude-code": "a", "pair/novox/postgres-database/g14/umami": "b"}} - raw := `{"node":"ace","module":"claude-code","provisioning":{"templates":{"source":"x","digest":"y"},` + - `"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres",` + - `"at":"10.7.0.1","from":"novox","as":"mesh_ace_agent"}]}}` - if err := bus.Put(context.Background(), AssignmentsBucket, "ace.claude-code", []byte(raw)); err != nil { + var pp catalogue.PlanProvision + if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+ + `"at":"10.7.0.1","from":"novox","as":"mesh_ace_agent"}`), &pp); err != nil { t.Fatal(err) } + planned(t, bus, "ace", "claude-code", pp) assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil { t.Fatal(err) @@ -596,12 +643,12 @@ func TestAConsumersEntryCarriesTheNamesOfItsPlan(t *testing.T) { bus, m := newStandIn(), &maker{} v := &vault{values: map[string]string{ "pair/novox/postgres-database/ace/gitea": "a", "pair/novox/postgres-database/g14/umami": "b"}} - raw := `{"node":"ace","module":"gitea","provisioning":{"templates":{"source":"x","digest":"y"},` + - `"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres",` + - `"names":{"name":"git.example.org","internal-name":"git.mesh","name-web":"web.example.org"}}]}}` - if err := bus.Put(context.Background(), AssignmentsBucket, "ace.gitea", []byte(raw)); err != nil { + var pp catalogue.PlanProvision + if err := json.Unmarshal([]byte(`{"provision":"postgres-database","provider":"novox","provider-module":"postgres",`+ + `"names":{"name":"git.example.org","internal-name":"git.mesh","name-web":"web.example.org"}}`), &pp); err != nil { t.Fatal(err) } + planned(t, bus, "ace", "gitea", pp) assign(t, bus, "g14", "umami", bind("postgres-database", "", "novox", "postgres")) if _, err := postgres(bus, v, m).Pass(context.Background()); err != nil { t.Fatal(err) @@ -719,6 +766,26 @@ func TestThePlansCredentialFormWinsOverTheOffers(t *testing.T) { if e := run(Shared, nil); e.VaultName != "provision/novox/postgres-database" { t.Fatalf("the offer's form is the default: %+v", e) } + { + // An offer that declares no credential is answered with none whatever form the plan says (the controller writes + // "pair" for such an offer): no vault name, and the Vault is never asked, so a provider with no Vault works (issue 825, + // finding 4; issue 923). + bus := newStandIn() + b := bind("route", "", "novox", "route-proxy") + b["credential-form"] = "pair" + assign(t, bus, "ace", "letta", b) + p := New(Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: func(string, ...any) {}, + Offers: map[string]Offer{"route": {Credential: NoCredential}}}) + out, err := p.Pass(context.Background()) + if err != nil || len(out) != 1 || out[0].Status != Answered { + t.Fatalf("%v %+v", err, out) + } + var e Entry + if err := json.Unmarshal(bus.get(BucketOf("route-proxy"), "ace.letta.route"), &e); err != nil || e.VaultName != "" || + e.User != "" { + t.Fatalf("%v %+v", err, e) + } + } } // An instance deletes only an entry it wrote, and only while the entry is as it read it: one that another instance of @@ -733,7 +800,7 @@ func TestAnEntryAnotherInstanceWroteAfterTheMoveStands(t *testing.T) { } // The consumer moves to the instance on shanks: A's pass reads its own entry and the unbind, and while it lets // the consumer go, B writes the consumer's entry. - _ = bus.Delete(context.Background(), AssignmentsBucket, "ace.letta") + retire(t, bus, "ace", "letta", "postgres-database", "novox", "postgres") theirs, _ := json.Marshal(Entry{Provider: "shanks", ProviderModule: "postgres", Provision: "postgres-database", Consumer: "ace", Module: "letta", VaultName: "pair/shanks/postgres-database/ace/letta"}) a.opts.Gone = func(ctx context.Context, e Entry) error { @@ -781,23 +848,83 @@ func TestAFailedMakeIsSaidWithoutTheCredentialInAnyOfItsForms(t *testing.T) { } } -// An offer that declares no credential is answered with none whatever form the plan says (the controller writes -// "pair" for such an offer): no vault name, and the Vault is never asked, so a provider with no Vault works (issue 825, -// finding 4; issue 923). -func TestAnOfferWithNoCredentialIgnoresThePlansForm(t *testing.T) { - bus := newStandIn() - b := bind("route", "", "novox", "route-proxy") - b["credential-form"] = "pair" - assign(t, bus, "ace", "letta", b) - p := New(Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: func(string, ...any) {}, - Offers: map[string]Offer{"route": {Credential: NoCredential}}}) - out, err := p.Pass(context.Background()) - if err != nil || len(out) != 1 || out[0].Status != Answered { - t.Fatalf("%v %+v", err, out) +// novox/hq ADR 0570 clauses 3 and 5 (issue 1954 L6-P3, P4): a consumer the seat mesh-provisioner plans is bound by its +// plan and its provision key, never by its assignment. Its entry is made only once its key is asked and every provision +// it depends on is provided, and says provided; a plan gone retires nothing by itself; a retiring key is let go only on +// the operator's yes. +func TestTheSeatsKeysSayWhenAProviderActsAndWhenItLetsGo(t *testing.T) { + ctx := context.Background() + bus, m := newStandIn(), &maker{} + v := &vault{values: map[string]string{"pair/novox/postgres-database/ace/letta": "s1"}} + // The same consumer's assignment names another provider: the seat's plan wins. + assign(t, bus, "ace", "letta", bind("postgres-database", "", "shanks", "postgres")) + plan, _ := provisions.EncodePlan(provisions.Plan{Node: "ace", Module: "letta", Plan: catalogue.Plan{ + Provisions: []catalogue.PlanProvision{{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres"}}}}) + _ = bus.Put(ctx, provisions.PlansBucket, "ace.letta", plan) + key := "ace.letta.postgres-database" + put := func(k string, state string, depends []string, warrant *provisions.Warrant) { + pr := provisions.Provision{Provision: "postgres-database", Consumer: provisions.End{Module: "letta", Node: "ace"}, + Provider: provisions.End{Module: "postgres", Node: "novox"}, DependsOn: depends, State: state, Warrant: warrant} + if k != key { + _, _, pr.Provision, _, _ = provisions.ParseKey(k) + pr.Provider = provisions.End{Module: "mesh-vault", Node: "novox"} + } + if state != provisions.Asked && state != provisions.Provided { + pr.Why = "said" + } + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + _ = bus.Put(ctx, provisions.ProvisionsBucket, k, raw) } - var e Entry - if err := json.Unmarshal(bus.get(BucketOf("route-proxy"), "ace.letta.route"), &e); err != nil || e.VaultName != "" || - e.User != "" { - t.Fatalf("%v %+v", err, e) + p := postgres(bus, v, m) + steps := []struct { + name string + before func() + status string + says string + entry bool + }{ + {name: "no key yet", status: Waiting, says: "holds no key"}, + {name: "a dependency asked", before: func() { + put("ace.letta.secret", provisions.Asked, nil, nil) + put(key, provisions.Asked, []string{"ace.letta.secret"}, nil) + }, status: Waiting, says: "depends on ace.letta.secret, which is asked"}, + {name: "the dependency provided", before: func() { put("ace.letta.secret", provisions.Provided, nil, nil) }, + status: Answered, entry: true}, + {name: "its plan gone: nothing let go by itself", before: func() { _ = bus.Delete(ctx, provisions.PlansBucket, "ace.letta") }, + entry: true}, + {name: "retiring, not yet answered", before: func() { put(key, provisions.Retiring, nil, nil) }, entry: true}, + {name: "retiring, the operator's yes", before: func() { + put(key, provisions.Retiring, nil, &provisions.Warrant{Ask: "a-1", Option: "yes", By: "the operator"}) + }, status: Removed}, + } + for _, s := range steps { + if s.before != nil { + s.before() + } + out, err := p.Pass(ctx) + if err != nil { + t.Fatalf("%s: %v", s.name, err) + } + var mine []Outcome + for _, o := range out { + if o.Key == key { + mine = append(mine, o) + } + } + if s.status != "" && (len(mine) != 1 || mine[0].Status != s.status || !strings.Contains(mine[0].Why, s.says)) { + t.Errorf("%s: %+v", s.name, mine) + } + if held := bus.get(BucketOf("postgres"), key) != nil; held != s.entry { + t.Errorf("%s: an entry held %v", s.name, held) + } + if s.entry && entryOf(t, bus, key).State != provisions.Provided { + t.Errorf("%s: the entry does not say provided", s.name) + } + } + if !slices.Equal(m.gone, []string{key}) || m.made[key] != "s1" { + t.Errorf("gone %v, made %v: let go once, on the warrant, and made from the vault's value", m.gone, m.made) } } diff --git a/go/provider/statebus_test.go b/go/provider/statebus_test.go index efe29de..1e5f391 100644 --- a/go/provider/statebus_test.go +++ b/go/provider/statebus_test.go @@ -10,6 +10,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // runtime drives a bundle over pipes as the runtime does: it reads what the bundle says and sends what the runtime would. @@ -55,8 +56,9 @@ func serve(t *testing.T, tools []stdio.Tool) *runtime { return &runtime{t: t, lines: bufio.NewScanner(outR), in: inW} } -// The bus of a moved provider maps its own bucket to the state `bound` and the assignments to the state -// `mesh-controller.assignments`, writes only the own bucket's state, and refuses any other bucket. +// The bus of a moved provider maps its own bucket to the state `bound` and the seat's plans to the state +// `mesh-provisioner.plans` (novox/hq ADR 0570 clause 2), writes only the own bucket's state, and refuses any other +// bucket, the controller's assignments among them. func TestStateBusMapsTheProvidersBucketsToTheRuntimesStates(t *testing.T) { bus := StateBus("postgres") rt := serve(t, []stdio.Tool{ @@ -72,7 +74,7 @@ func TestStateBusMapsTheProvidersBucketsToTheRuntimesStates(t *testing.T) { }) for i, c := range []struct{ bucket, state string }{ {BucketOf("postgres"), "bound"}, - {AssignmentsBucket, "mesh-controller.assignments"}, + {provisions.PlansBucket, "mesh-provisioner.plans"}, } { rt.call(i+1, "watch", map[string]any{"bucket": c.bucket}) w := rt.next() @@ -97,7 +99,7 @@ func TestStateBusMapsTheProvidersBucketsToTheRuntimesStates(t *testing.T) { rt.send(map[string]any{"jsonrpc": "2.0", "id": d["id"], "result": map[string]any{}}) rt.next() - for _, bucket := range []string{"other_bound", "elsewhere"} { + for _, bucket := range []string{"other_bound", "elsewhere", AssignmentsBucket} { if err := bus.Watch(context.Background(), bucket, func(string, []byte) {}); err == nil || !strings.Contains(err.Error(), "not a bucket") { t.Fatalf("a watch of %s gave %v", bucket, err) } @@ -118,7 +120,7 @@ func TestStateBusWatchCallsItsHandlerNoMoreOnceItsContextEnded(t *testing.T) { handled := make(chan string, 8) rt := serve(t, []stdio.Tool{ {Name: "start", Run: func(map[string]any) (any, error) { - return "started", bus.Watch(ctx, AssignmentsBucket, func(key string, value []byte) { + return "started", bus.Watch(ctx, provisions.PlansBucket, func(key string, value []byte) { handled <- key + "=" + string(value) }) }}, From 161fba56aea31c9edcfba3a4fe4b111c4018ddea Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:42:21 +0100 Subject: [PATCH 30/52] Hold the provisioner's records to one writer in the credential table: only mesh-provisioner writes provisions and plans, readers only read, and it writes none of the grants part's states (issue 1954 L6-P5, ADR 0570 clauses 1, 8) --- go/buscompose/buscompose_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index dc82f65..97bb99a 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -603,6 +603,12 @@ func healthChecks(t *testing.T) { {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, + // ADR 0570 clauses 1, 8 (issue 1954 L6-P5): mesh-provisioner keeps provisions and plans; the vault and a provider + // read them. + {Module: "mesh-provisioner", State: []Bucket{bucket("mesh-provisioner", "provisions"), bucket("mesh-provisioner", "plans"), + bucket("mesh-provisioner", "registry")}}, + {Module: "mesh-vault", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}}, + {Module: "postgres", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}}, {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools", @@ -613,8 +619,11 @@ func healthChecks(t *testing.T) { t.Fatal(err) } may := map[string][]string{ - "a.mesh-conditions": {"$KV.mesh-conditions_conditions.machine.g14.silent"}, - "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, + "a.mesh-conditions": {"$KV.mesh-conditions_conditions.machine.g14.silent"}, + "a.mesh-provisioner": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, + "a.mesh-vault": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", + "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, + "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.INFO.SEAT_TELEGRAM_SENDER", @@ -626,6 +635,11 @@ func healthChecks(t *testing.T) { // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, + // Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570 + // clauses 1, 8). + "a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"}, + "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, + "a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, // Nor mesh-hand-acts' log and warrants, which it alone writes (ADR 0565 clause 10). "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.mesh-hand-acts_warrants.x", "mesh.log.mesh-hand-acts.acts.x"}, From f701db05162ce664191c710023797aed77a25f94 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:43:56 +0100 Subject: [PATCH 31/52] Key a provision's credential as the vault's own provision secret- and say what a provision depends on, so a provider waits for its credential and its own requirements (issue 1954, ADR 0570 clauses 3, 6) --- go/catalogue/manifest.go | 10 +++++++ go/provisions/provisions.go | 48 ++++++++++++++++++++++++++++++++ go/provisions/provisions_test.go | 24 ++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index 1423d11..0be481b 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -1775,6 +1775,10 @@ func (m Manifest) marshalFields() ([]byte, error) { return json.Marshal(keys) } +// CredentialProvisionPrefix starts the name of a provision's credential, the vault's provision beside it (novox/hq ADR +// 0570 clause 6): `secret-`. +const CredentialProvisionPrefix = "secret-" + // SecretFile is one file a module is given a credential in: the local name it goes by inside // the module (empty for the ordinary one-file case, where the requirement's name serves) and where. type SecretFile struct { @@ -1937,6 +1941,12 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { if !name.MatchString(p) { problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p)) } + // A provision's credential is a provision of its own, `secret-` (novox/hq ADR 0570 clause 6, the + // lead 2026-10-29): no module provides a name of that form, so the two keys never meet. + if strings.HasPrefix(p, CredentialProvisionPrefix) { + problems = append(problems, fmt.Sprintf("%s provides %q; a name starting %q is the mesh's for a provision's "+ + "credential, kept by the vault (novox/hq ADR 0570)", m.Module, p, CredentialProvisionPrefix)) + } // A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it // leaves room for the shortest identity the mesh makes, or it would refuse every consumer. if id := offer.Identity; id != nil && !id.None { diff --git a/go/provisions/provisions.go b/go/provisions/provisions.go index 18e1f17..a46753d 100644 --- a/go/provisions/provisions.go +++ b/go/provisions/provisions.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "regexp" + "sort" "strings" "time" @@ -121,6 +122,53 @@ func VaultName(form string, p Provision) string { return "" } +// VaultModule is the provider of every provision's credential (novox/hq ADR 0570 clauses 1, 6). +const VaultModule = "mesh-vault" + +// CredentialKey is the key of a provision's credential, itself a provision whose provider is the vault (clause 6; the +// key form the lead settled on 2026-10-29): `..secret-[.]` for a value of the +// consumer's own (pair, issued, given), and `..secret-` for the provider's one +// shared value. False for a form with no vault value. +func CredentialKey(form string, consumer, provider End, provision, local string) (string, bool) { + name := catalogue.CredentialProvisionPrefix + provision + switch form { + case FormPair, FormIssued, FormGiven: + k, err := Key(consumer.Node, consumer.Module, name, local) + return k, err == nil + case FormShared: + k, err := Key(provider.Node, provider.Module, name, "") + return k, err == nil + } + return "", false +} + +// DependsOn is what a provision waits for before its provider makes it (clause 3): its credential's key, where its form +// has a vault value, and the key of every provision its provider module itself requires on its own node, read from +// the provider's plan there (nil when the provider is not handed and has none). Sorted, none twice. +func DependsOn(consumer End, entry catalogue.PlanProvision, providerPlan *catalogue.Plan) []string { + seen := map[string]bool{} + var out []string + add := func(k string) { + if k != "" && !seen[k] { + seen[k] = true + out = append(out, k) + } + } + provider := End{Module: entry.ProviderModule, Node: entry.Provider} + if k, ok := CredentialKey(entry.CredentialForm, consumer, provider, entry.Provision, entry.Local); ok { + add(k) + } + if providerPlan != nil { + for _, own := range providerPlan.Provisions { + if k, err := Key(provider.Node, provider.Module, own.Provision, own.Local); err == nil { + add(k) + } + } + } + sort.Strings(out) + return out +} + // End is one end of a provision: a module on a node. type End struct { Module string `json:"module"` diff --git a/go/provisions/provisions_test.go b/go/provisions/provisions_test.go index c720f42..de95a80 100644 --- a/go/provisions/provisions_test.go +++ b/go/provisions/provisions_test.go @@ -82,6 +82,30 @@ func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { if got := VaultName(FormPair, Provision{Provision: "p", Local: "r", Provider: End{Node: "a"}, Consumer: End{Node: "b", Module: "m"}}); got != "pair/a/p/b/m/r" { t.Errorf("a local name's pair is %q", got) } + // A provision waits for its credential, the vault's provision `secret-`, and for what its provider itself + // requires on its node (clause 3; the key form the lead settled on 2026-10-29). + keycloak, postgres := End{Module: "keycloak", Node: "novox"}, End{Module: "postgres", Node: "ace"} + providerPlan := &catalogue.Plan{Provisions: []catalogue.PlanProvision{{Provision: "backup-target", Provider: "ace", ProviderModule: "restic"}}} + for _, c := range []struct { + form string + plan *catalogue.Plan + want string + }{ + {FormPair, nil, "novox.keycloak.secret-postgres-database"}, + {FormShared, nil, "ace.postgres.secret-postgres-database"}, + {FormPair, providerPlan, "ace.postgres.backup-target,novox.keycloak.secret-postgres-database"}, + {"", nil, ""}, + } { + entry := catalogue.PlanProvision{Provision: "postgres-database", Provider: postgres.Node, ProviderModule: postgres.Module, + CredentialForm: c.form} + if got := strings.Join(DependsOn(keycloak, entry, c.plan), ","); got != c.want { + t.Errorf("%q depends on %q, want %q", c.form, got, c.want) + } + } + if _, err := catalogue.ParseManifest([]byte(`{"module":"x","version":"1","provides":[{"name":"secret-db","scope":"mesh"}]}`)); err == nil || + !strings.Contains(err.Error(), "secret-") { + t.Errorf("a provision named secret-*: %v", err) + } if ProvisionsBucket != "mesh-provisioner_provisions" || PlansRead != "mesh-provisioner.plans" { t.Errorf("the states are %s and %s", ProvisionsBucket, PlansRead) } From dedd988fc929a7c9e92874721651e1287eb2657a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:44:49 +0100 Subject: [PATCH 32/52] Export a bus in memory and a mesh whose plan hands a module with a provision, so mesh-provisioner's whole test composes a real plan (issue 1954 H5) --- go/compose/frombus/frombustest/frombustest.go | 183 ++++++++++++++++ .../frombus/frombustest/frombustest_test.go | 26 +++ ...5aa5e6-mesh-provisioner-client.module.json | 60 ++++++ .../backups.conf | 2 + .../catalogue-c05aa5e6-restic.files/data.conf | 2 + .../retention.conf | 6 + .../catalogue-c05aa5e6-restic.module.json | 199 ++++++++++++++++++ 7 files changed, 478 insertions(+) create mode 100644 go/compose/frombus/frombustest/frombustest.go create mode 100644 go/compose/frombus/frombustest/frombustest_test.go create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf create mode 100644 go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json diff --git a/go/compose/frombus/frombustest/frombustest.go b/go/compose/frombus/frombustest/frombustest.go new file mode 100644 index 0000000..3bfc971 --- /dev/null +++ b/go/compose/frombus/frombustest/frombustest.go @@ -0,0 +1,183 @@ +// Package frombustest is a bus in memory for frombus's Reader, and a mesh on it whose composition hands a module to the +// provisioner with a provision to make, so a module that composes plans from the bus (mesh-provisioner, novox/hq ADR +// 0570 clause 2) tests against a real plan rather than one written by hand. The manifests are the catalogue's (mesh-catalog +// c05aa5e6: restic and the provisioner's client) as frombus's own tests read them, restic given one requirement more. +package frombustest + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "embed" + "encoding/json" + "errors" + "strings" + + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +//go:embed testdata +var testdata embed.FS + +// Bus is a frombus.Reader over values held in memory. A credential asked is answered held when Held is set. +type Bus struct { + Placement map[string]delivery.Placement + Record map[string]delivery.NodeRecord + Module map[string]delivery.Module + Manifests map[string][]byte + Value map[string]map[string]any // by node.module + Own []compose.OwnSecretRow + Builds map[string]*delivery.BuilderBuild // by " " + Archives map[string][]byte // by digest + Serves map[string]map[string]any // another node's values, by node.module + Held bool +} + +var _ frombus.Reader = (*Bus)(nil) + +func (b *Bus) Placements(context.Context) (map[string]delivery.Placement, error) { + return b.Placement, nil +} +func (b *Bus) Records(context.Context) (map[string]delivery.NodeRecord, error) { return b.Record, nil } +func (b *Bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.Module, nil } +func (b *Bus) VaultAuthority(context.Context) string { return "" } + +func (b *Bus) Manifest(_ context.Context, digest string) ([]byte, error) { + raw, ok := b.Manifests[digest] + if !ok { + return nil, errors.New("no such manifest") + } + return raw, nil +} + +func (b *Bus) Values(_ context.Context, node, module string) (map[string]any, error) { + return b.Value[node+"."+module], nil +} + +func (b *Bus) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return b.Own, nil } + +func (b *Bus) Credentials(_ context.Context, _ string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + var out []compose.Credential + var issued []compose.Issued + for _, k := range append(append([]compose.CredentialKey{}, asConsumer...), asProvider...) { + if b.Held { + out = append(out, compose.Credential{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule, + Provider: k.Provider, Local: k.Local, State: compose.CredentialHeld, Sealed: `"box"`}) + issued = append(issued, compose.Issued{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule, + Provider: k.Provider, Local: k.Local, ForProvider: `"box"`}) + } + } + return out, issued, nil +} + +func (b *Bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { + return b.Builds[module+" "+fp], nil +} + +func (b *Bus) Archive(_ context.Context, _, digest string) ([]byte, error) { + body, ok := b.Archives[digest] + if !ok { + return nil, errors.New("the store holds no such blob") + } + return body, nil +} + +func (b *Bus) Served(_ context.Context, node, module string, keys []string) (map[string]any, error) { + out := map[string]any{} + for _, k := range keys { + if v, ok := b.Serves[node+"."+module][k]; ok { + out[k] = v + } + } + return out, nil +} + +// Node is the one node of the mesh Provisioned composes. +const Node = "n1" + +// Provisioned is one node, n1, that runs the artifact store, a postgres providing postgres-database, the provisioner's +// client and restic: restic built and handed to the provisioner, and requiring postgres-database. Its plans (frombus.Plans +// with Scope{Node: Node}) hand restic, whose plan names that provision with postgres on n1 as its provider. +func Provisioned() (*Bus, error) { + read := func(name string) ([]byte, error) { return testdata.ReadFile("testdata/" + name) } + resticRaw, err := read("catalogue-c05aa5e6-restic.module.json") + if err != nil { + return nil, err + } + var restic map[string]any + if err := json.Unmarshal(resticRaw, &restic); err != nil { + return nil, err + } + restic["requires"] = []string{"postgres-database"} + if resticRaw, err = json.Marshal(restic); err != nil { + return nil, err + } + clientRaw, err := read("catalogue-c05aa5e6-mesh-provisioner-client.module.json") + if err != nil { + return nil, err + } + store := []byte(`{"module":"store","version":"1","provides":[{"name":"artifact-store","scope":"mesh"}],` + + `"serves":{"artifact-store":{"port":5000}}}`) + postgres := []byte(`{"module":"postgres","version":"1","provides":[{"name":"postgres-database","scope":"mesh"}],` + + `"serves":{"postgres-database":{"port":5432}}}`) + + var packed bytes.Buffer + zw := gzip.NewWriter(&packed) + tw := tar.NewWriter(zw) + for _, f := range []string{"backups.conf", "data.conf", "retention.conf"} { + text, err := read("catalogue-c05aa5e6-restic.files/" + f) + if err != nil { + return nil, err + } + if err := tw.WriteHeader(&tar.Header{Name: "./" + f, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil { + return nil, err + } + if _, err := tw.Write(text); err != nil { + return nil, err + } + } + if err := tw.Close(); err != nil { + return nil, err + } + if err := zw.Close(); err != nil { + return nil, err + } + archive := delivery.ManifestDigest(packed.Bytes()) + digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } + fps := FPs() + b := &Bus{ + Placement: map[string]delivery.Placement{Node: {Node: Node, + Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1", Account: "operator", AccountHome: "/home/operator"}, + Modules: []delivery.PlacedModule{{Module: "store"}, {Module: "postgres"}, {Module: "mesh-provisioner-client"}, {Module: "restic"}}}}, + Record: map[string]delivery.NodeRecord{Node: {Node: Node, TunnelKey: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3Q=", + SealingKey: "sk", Contract: compose.HandedContract, Capabilities: []string{"systemd", "package-manager"}}}, + Module: map[string]delivery.Module{}, Manifests: map[string][]byte{}, + Own: []compose.OwnSecretRow{{Node: Node, Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}, + Builds: map[string]*delivery.BuilderBuild{ + "restic " + fps["restic"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: []delivery.BuiltArtifact{ + {Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, + {Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archive, Digest: archive}}}, + "mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw), + Artifacts: []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", + Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}}}, + "store " + fps["store"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)}, + "postgres " + fps["postgres"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(postgres)}, + }, + Archives: map[string][]byte{archive: packed.Bytes()}, + Held: true, + } + for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store, "postgres": postgres} { + b.Module[module] = delivery.Module{FP2: fps[module], Manifest: delivery.ManifestDigest(raw)} + b.Manifests[delivery.ManifestDigest(raw)] = raw + } + return b, nil +} + +// FPs are the fingerprints Provisioned's modules are built at. +func FPs() map[string]string { + return map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64), + "store": "src2:" + strings.Repeat("5", 64), "postgres": "src2:" + strings.Repeat("8", 64)} +} diff --git a/go/compose/frombus/frombustest/frombustest_test.go b/go/compose/frombus/frombustest/frombustest_test.go new file mode 100644 index 0000000..edb8388 --- /dev/null +++ b/go/compose/frombus/frombustest/frombustest_test.go @@ -0,0 +1,26 @@ +package frombustest + +import ( + "context" + "testing" + + "git.novox.be/novox/mesh-sdk/go/compose/frombus" +) + +// ADR 0570 clause 2 (issue 1954; a3's H5): the in-memory mesh composes a plan from the bus that hands restic to the +// provisioner and names its postgres-database, provided by postgres on n1. +func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { + b, err := Provisioned() + if err != nil { + t.Fatal(err) + } + plans, err := frombus.Plans(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}) + if err != nil { + t.Fatal(err) + } + restic, handed := plans["restic"] + if !handed || len(restic.Provisions) != 1 || restic.Provisions[0].Provision != "postgres-database" || + restic.Provisions[0].Provider != Node || restic.Provisions[0].ProviderModule != "postgres" { + t.Fatalf("the plans are %+v", plans) + } +} diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json new file mode 100644 index 0000000..095128a --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-mesh-provisioner-client.module.json @@ -0,0 +1,60 @@ +{ + "module": "mesh-provisioner-client", + "version": "1", + "claims": [ + { + "name": "node-provisioner", + "scope": "node" + } + ], + "own-secrets": { + "provisioner-broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-provisioner-client", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-provisioner-client", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-provisioner-client" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-provisioner-client", + "place": ".", + "unknown": "program" + }, + { + "id": "client", + "type": "process", + "name": "mesh-provisioner-client", + "artifact": "client", + "run": [ + "./mesh-provisioner-client" + ], + "env": { + "MESH_PROVISIONER_CREDENTIAL": "${dir:state}/broker", + "MESH_PROVISIONER_STATE": "${dir:state}" + }, + "user": "mesh-provisioner-client", + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "client", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-provisioner-client", + "binary": "mesh-provisioner-client" + } + ] + } +} diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf new file mode 100644 index 0000000..2f23f90 --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/backups.conf @@ -0,0 +1,2 @@ +# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit. +${backup:lines} \ No newline at end of file diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf new file mode 100644 index 0000000..f19e849 --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/data.conf @@ -0,0 +1,2 @@ +# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit. +${backup:data} \ No newline at end of file diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf new file mode 100644 index 0000000..cd7954c --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.files/retention.conf @@ -0,0 +1,6 @@ +# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily, +# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file. +daily=${config:keep-daily} +weekly=${config:keep-weekly} +monthly=${config:keep-monthly} +hand=${config:keep-hand-taken-days} diff --git a/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json new file mode 100644 index 0000000..ea5ec8b --- /dev/null +++ b/go/compose/frombus/frombustest/testdata/catalogue-c05aa5e6-restic.module.json @@ -0,0 +1,199 @@ +{ + "module": "restic", + "version": "1", + "moved": true, + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "emits": [ + "backed-up" + ], + "tools": [ + "backup_health", + "backup_delete_retired", + "backup_deleted" + ], + "own-secrets": { + "repository": "${dir:state}/repository.secret" + }, + "data": { + "own": [ + { + "id": "repository", + "path": "${dir:repository}", + "class": "rebuildable", + "backup": "none", + "why": "the restore points themselves: a copy of data kept elsewhere on this machine, never the only copy of anything; lost, the history goes and nothing live does" + }, + { + "id": "state", + "path": "${dir:state}", + "class": "cache", + "why": "what the last nights and measurements were; the next night writes it again" + } + ] + }, + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 + }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } + }, + "resources": [ + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, + { + "id": "repository", + "type": "directory", + "mode": "0700", + "unknown": "program" + }, + { + "id": "declared", + "type": "file", + "path": "${dir:state}/backups.conf", + "mode": "0600", + "template": "files/backups.conf" + }, + { + "id": "data-declared", + "type": "file", + "path": "${dir:state}/data.conf", + "mode": "0600", + "template": "files/data.conf" + }, + { + "id": "retention", + "type": "file", + "path": "${dir:state}/retention.conf", + "mode": "0644", + "trusted": true, + "template": "files/retention.conf" + }, + { + "id": "tool", + "type": "package", + "package": "restic" + }, + { + "id": "sqlite", + "type": "package", + "package": "sqlite" + }, + { + "id": "night", + "type": "process", + "name": "restic-night", + "artifact": "tools", + "run": [ + "./restic-backups", + "night" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" + } + }, + { + "id": "measure", + "type": "process", + "name": "restic-measure", + "artifact": "tools", + "run": [ + "./restic-backups", + "measure" + ], + "user": "${machine:account}", + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + }, + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/restic-backups", + "binary": "restic-backups", + "loads": [ + "restic-backups" + ], + "env": { + "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", + "MESH_BACKUP_REPOSITORY": "${dir:repository}", + "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" + } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} From 6b8c6943811ac3ec4b607b9552d8c8beec2f07f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:47:32 +0100 Subject: [PATCH 33/52] Make every offer say its credential's form, pair or none where it names no own secret or issuer, and read no default, so the vault never mints a login for a provider that takes none (issue 1954, ADR 0570) --- go/catalogue/handed_providers_test.go | 22 ++++++++++++++--- go/catalogue/manifest.go | 35 +++++++++++++++++++++++++-- go/catalogue/seats_declared.go | 4 +++ go/catalogue/seats_declared_test.go | 4 +-- go/catalogue/seats_test.go | 2 +- go/provisions/provisions.go | 3 +++ go/provisions/provisions_test.go | 4 ++- 7 files changed, 65 insertions(+), 9 deletions(-) diff --git a/go/catalogue/handed_providers_test.go b/go/catalogue/handed_providers_test.go index 22744f8..6541c74 100644 --- a/go/catalogue/handed_providers_test.go +++ b/go/catalogue/handed_providers_test.go @@ -42,9 +42,10 @@ func consumerOn(t *testing.T, m Manifest, w World) Resolution { node := workstation() node.At = "workstation.internal" stores := shelf(m, - Manifest{Module: "store", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh}}, + Manifest{Module: "store", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh, + Credential: &OfferCredential{Form: CredentialFormPair}}}, Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}}, - Manifest{Module: "store-two", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh}}, + Manifest{Module: "store-two", Version: "1", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh, Credential: &OfferCredential{Form: CredentialFormPair}}}, Serves: map[string]map[string]any{"postgres-database": {"port": 5434}}}) r, err := Resolve(stores, []string{m.Module}, node, w) if err != nil { @@ -130,7 +131,10 @@ func TestAHandedModulesPlanNamesTheCredentialFormOfItsProvidersOffer(t *testing. return mustParse(t, raw+`}`) } for credential, want := range map[string]string{ - ``: CredentialFormPair, + // No default (the lead, 2026-10-29): an offer that says nothing has no form, and the catalogue refuses it. + ``: "", + `,"credential":{"form":"pair"}`: CredentialFormPair, + `,"credential":{"form":"none"}`: CredentialFormNone, `,"credential":{"own":"admin"}`: CredentialFormShared, `,"credential":{"own":"token"}`: CredentialFormGiven, `,"credential":{"issued-by":"outside"}`: CredentialFormGiven, @@ -153,6 +157,18 @@ func TestAHandedModulesPlanNamesTheCredentialFormOfItsProvidersOffer(t *testing. if got := (Manifest{Module: "x"}).CredentialFormOf("postgres-database"); got != "" { t.Errorf("a module offering nothing has the credential form %q", got) } + // The catalogue refuses an offer saying no form, and a form beside an own secret or not pair or none. + for credential, refused := range map[string]string{``: "says no credential form", `,"credential":{}`: "says no credential form", + `,"credential":{"form":"pair","own":"admin"}`: "beside an own secret", `,"credential":{"form":"maybe"}`: `not "pair" or "none"`, + `,"credential":{"form":"none"}`: ""} { + raw := `{"module":"store","version":"1","provides":[{"name":"postgres-database","scope":"mesh"` + credential + `}]}` + var m Manifest + _ = json.Unmarshal([]byte(raw), &m) + problems := strings.Join(CredentialFormProblems(m), "; ") + if (refused == "") != (problems == "") || !strings.Contains(problems, refused) { + t.Errorf("%q: %q, want %q", credential, problems, refused) + } + } } // A consumer's plan entry carries what its manifest contributes to that provision (issue 776), so the provider's diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index 0be481b..1d5242f 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -287,6 +287,10 @@ type OfferCredential struct { Own string `json:"own,omitempty"` IssuedBy string `json:"issued-by,omitempty"` Tool string `json:"tool,omitempty"` + // Form says, for an offer that names no own secret and no issuer, which it is: CredentialFormPair, a login of + // each consumer's own, or CredentialFormNone, no credential at all (the lead, 2026-10-29, for novox/hq ADR 0570): + // never left to a default. + Form string `json:"form,omitempty"` } // IssuedBySoftware says a provision's credential is issued by its provider's software (novox/hq ADR 0305). @@ -362,6 +366,9 @@ const ( // CredentialFormGiven is given by a person (novox/hq ADR 0337): the offer's credential, or the own secret it // shares, is issued outside the mesh; the mesh never makes it. CredentialFormGiven = "given" + // CredentialFormNone is an offer whose consumers present no credential: its outcome is the provider's have, and + // the vault mints nothing for it. + CredentialFormNone = "none" ) // CredentialFormOf is the credential form this module's offer of a provision declares, read from the offer's @@ -375,7 +382,7 @@ func (m Manifest) CredentialFormOf(provision string) string { } switch c := o.Credential; { case c == nil: - return CredentialFormPair + return "" // undeclared: refused at the catalogue (CredentialFormProblems), never read as a pair case c.IssuedBy == IssuedOutside: return CredentialFormGiven case c.IssuedBy == IssuedBySoftware: @@ -385,11 +392,33 @@ func (m Manifest) CredentialFormOf(provision string) string { case c.Own != "": return CredentialFormShared } - return CredentialFormPair + return o.Credential.Form } return "" } +// CredentialFormProblems refuses an offer that does not say its credential's form (the lead, 2026-10-29): an own +// secret, an issuer, or `form` pair or none, and `form` only where neither of the others is said. +func CredentialFormProblems(m Manifest) []string { + var problems []string + for _, o := range m.Provides { + c := o.Credential + switch { + case c == nil || (c.Own == "" && c.IssuedBy == "" && c.Form == ""): + problems = append(problems, fmt.Sprintf("%s provides %q and says no credential form: say `credential` with "+ + "`form` pair (a login of each consumer's own) or none (no credential), or the own secret or issuer it is", + m.Module, o.Name)) + case c.Form != "" && (c.Own != "" || c.IssuedBy != ""): + problems = append(problems, fmt.Sprintf("%s provides %q with `form` beside an own secret or an issuer; `form` "+ + "says only pair or none", m.Module, o.Name)) + case c.Form != "" && c.Form != CredentialFormPair && c.Form != CredentialFormNone: + problems = append(problems, fmt.Sprintf("%s provides %q with the form %q, not %q or %q", m.Module, o.Name, + c.Form, CredentialFormPair, CredentialFormNone)) + } + } + return problems +} + // ProvisionsSharing is every provision of this module whose credential is the named own secret. func (m Manifest) ProvisionsSharing(own string) []string { var out []string @@ -1995,6 +2024,8 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { } else if offer.Credential != nil { own, declared := m.OwnSecrets[offer.Credential.Own] switch { + case offer.Credential.Own == "" && offer.Credential.Form != "": + // The form alone, pair or none (CredentialFormProblems judges it). case offer.Credential.Own == "": problems = append(problems, fmt.Sprintf( "%s provides %q with a credential that names no own secret", m.Module, p)) diff --git a/go/catalogue/seats_declared.go b/go/catalogue/seats_declared.go index 9165103..90087e2 100644 --- a/go/catalogue/seats_declared.go +++ b/go/catalogue/seats_declared.go @@ -282,6 +282,10 @@ func CatalogueProblems(shelf Shelf) []string { problems = append(problems, RestrictedHearingProblems(shelf)...) // A work-queue stream is never asked for by another module's consumes (novox/hq ADR 0436 §6). problems = append(problems, WorkQueueConsumedElsewhereProblems(shelf)...) + // Every offer says its credential's form, never left to a default (the lead, 2026-10-29, for novox/hq ADR 0570). + for _, module := range shelfOrder(shelf) { + problems = append(problems, CredentialFormProblems(shelf[module])...) + } // Who declares what, and who declared it first. declaredBy := map[string]string{} diff --git a/go/catalogue/seats_declared_test.go b/go/catalogue/seats_declared_test.go index 33b3ba6..25a8d0b 100644 --- a/go/catalogue/seats_declared_test.go +++ b/go/catalogue/seats_declared_test.go @@ -109,7 +109,7 @@ func TestAClaimMustMatchTheDeclaredScope(t *testing.T) { func TestTheMeshsOwnSeatsAreStillClaimable(t *testing.T) { // It delivers the bus, so its holder provides the bus — the rule this check now enforces. m := Manifest{Module: "nats", - Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh}}, + Provides: []Offer{{Name: "mesh-bus", Scope: ScopeMesh, Credential: &OfferCredential{Form: CredentialFormNone}}}, Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}} if got := problemsFor(t, Shelf{"nats": m}); got != "" { t.Fatalf("a mesh seat was refused by the derived check: %s", got) @@ -186,7 +186,7 @@ func TestTheParserDoesNotJudgeWhatOnlyTheStoreKnows(t *testing.T) { // A store whose bus seat delivers something this module does provide. UseSeats([]Seat{{Name: "mesh-broker", Scope: ScopeMesh, Delivers: "mesh-bus", Decision: "test"}}) raw := []byte(`{"module":"a-bus","version":"1",` + - `"provides":[{"name":"mesh-bus","scope":"mesh"}],` + + `"provides":[{"name":"mesh-bus","scope":"mesh","credential":{"form":"none"}}],` + `"claims":[{"name":"mesh-broker","scope":"mesh"}]}`) m, err := ParseManifest(raw) diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index ad001bb..828464e 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -148,7 +148,7 @@ func TestTheUplinkIsANodeSeatThatDeliversNothing(t *testing.T) { } func claimed(claims string) []byte { - return []byte(`{"module":"thing","version":"1","provides":[{"name":"npm-package-registry","scope":"mesh"}],"claims":` + claims + `}`) + return []byte(`{"module":"thing","version":"1","provides":[{"name":"npm-package-registry","scope":"mesh","credential":{"form":"none"}}],"claims":` + claims + `}`) } // **The refusal moved, it did not go** (novox/hq ADR 0118, superseding 0110). A module may now diff --git a/go/provisions/provisions.go b/go/provisions/provisions.go index a46753d..ca8d914 100644 --- a/go/provisions/provisions.go +++ b/go/provisions/provisions.go @@ -99,6 +99,9 @@ const ( FormShared = "shared" FormIssued = "issued" FormGiven = "given" + // FormNone is an offer whose consumers present no credential (catalogue.CredentialFormNone): the vault mints nothing, + // and the provision depends on no credential. + FormNone = "none" ) // VaultName is the vault's name of a provision's credential in form, as its verb `own` names it and the provider asks diff --git a/go/provisions/provisions_test.go b/go/provisions/provisions_test.go index de95a80..e0ba87a 100644 --- a/go/provisions/provisions_test.go +++ b/go/provisions/provisions_test.go @@ -74,7 +74,7 @@ func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { } // A reference's id is the vault's name as its verb own names it, in each form (the host agent, 2026-10-29). for form, want := range map[string]string{FormPair: "pair/novox/postgres-database/novox/keycloak", FormShared: "provision/novox/postgres-database", - FormIssued: "provision/novox/postgres-database/novox/keycloak", "none": ""} { + FormIssued: "provision/novox/postgres-database/novox/keycloak", FormNone: ""} { if got := VaultName(form, db); got != want { t.Errorf("the %s name is %q", form, got) } @@ -95,6 +95,8 @@ func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { {FormShared, nil, "ace.postgres.secret-postgres-database"}, {FormPair, providerPlan, "ace.postgres.backup-target,novox.keycloak.secret-postgres-database"}, {"", nil, ""}, + // An offer with no credential (the lead, 2026-10-29): nothing minted, nothing to wait for. + {FormNone, nil, ""}, } { entry := catalogue.PlanProvision{Provision: "postgres-database", Provider: postgres.Node, ProviderModule: postgres.Module, CredentialForm: c.form} From 58a83249c80b158c6a074224f56a6e2b3b8bf23d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:53:15 +0100 Subject: [PATCH 34/52] Give a provision the vault itself provides no credential beside it, since its offer is the value (issue 1954, ADR 0570 clause 6) --- go/provisions/provisions.go | 5 ++++- go/provisions/provisions_test.go | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/go/provisions/provisions.go b/go/provisions/provisions.go index ca8d914..1e7a4fb 100644 --- a/go/provisions/provisions.go +++ b/go/provisions/provisions.go @@ -131,8 +131,11 @@ const VaultModule = "mesh-vault" // CredentialKey is the key of a provision's credential, itself a provision whose provider is the vault (clause 6; the // key form the lead settled on 2026-10-29): `..secret-[.]` for a value of the // consumer's own (pair, issued, given), and `..secret-` for the provider's one -// shared value. False for a form with no vault value. +// shared value. False for a form with no vault value, and for a provision the vault itself provides. func CredentialKey(form string, consumer, provider End, provision, local string) (string, bool) { + if provider.Module == VaultModule { + return "", false // the vault's own offer is the value: it has no credential beside it (the lead, 2026-10-29) + } name := catalogue.CredentialProvisionPrefix + provision switch form { case FormPair, FormIssued, FormGiven: diff --git a/go/provisions/provisions_test.go b/go/provisions/provisions_test.go index e0ba87a..37a9796 100644 --- a/go/provisions/provisions_test.go +++ b/go/provisions/provisions_test.go @@ -104,6 +104,11 @@ func TestAProvisionIsAReferenceUnderItsConsumersKey(t *testing.T) { t.Errorf("%q depends on %q, want %q", c.form, got, c.want) } } + // The vault's own offer is the value: no secret-secret beside it. + vaultEntry := catalogue.PlanProvision{Provision: "secret", Provider: "novox", ProviderModule: VaultModule, CredentialForm: FormPair} + if got := DependsOn(keycloak, vaultEntry, nil); len(got) != 0 { + t.Errorf("the vault's secret depends on %v", got) + } if _, err := catalogue.ParseManifest([]byte(`{"module":"x","version":"1","provides":[{"name":"secret-db","scope":"mesh"}]}`)); err == nil || !strings.Contains(err.Error(), "secret-") { t.Errorf("a provision named secret-*: %v", err) From 94d0de5c19b6c2c2ee85548c2d0871c7d93a4321 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:58:05 +0100 Subject: [PATCH 35/52] Move the node's health report layout into the sdk and let each node write only its own key while the watchman reads them, so the six machine alerts leave the controller (issue 774) --- go/buscompose/buscompose_test.go | 12 ++- go/buscompose/engine.go | 6 ++ go/buscompose/writers.go | 4 + go/delivery/control_records_test.go | 18 +++- go/delivery/nodehealth.go | 146 ++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 go/delivery/nodehealth.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 97bb99a..c73690b 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -79,6 +79,8 @@ func engineBeyond(user, g string) bool { strings.Contains(g, "KV_"+EngineActsBucket) || strings.HasPrefix(g, "$KV."+EngineActsBucket+"."+node+".") || // And its node's declared packages (novox/hq ADR 0565 clause 6). strings.Contains(g, "KV_"+EnginePackagesBucket) || g == "$KV."+EnginePackagesBucket+"."+node || + // And its own health report (issue 774). + strings.Contains(g, "KV_"+delivery.NodeHealthBucket) || g == "$KV."+delivery.NodeHealthBucket+"."+node || // And the provisions of its node's modules, read (novox/hq ADR 0570 clause 6). strings.Contains(g, "KV_"+provisions.ProvisionsBucket) } @@ -601,7 +603,7 @@ func healthChecks(t *testing.T) { Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, - {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}}, + {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}}, {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, // ADR 0570 clauses 1, 8 (issue 1954 L6-P5): mesh-provisioner keeps provisions and plans; the vault and a provider // read them. @@ -675,6 +677,14 @@ func healthChecks(t *testing.T) { MayPublish(perms, "$KV.mesh-provisioner_provisions.a.keycloak.db")) { t.Errorf("node %s reads the provisions %v", u.Node, perms.Publish) } + // Issue 774: a node's engine writes its own health report's key and no other node's. + if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_health.a") || MayPublish(perms, "$KV.mesh-host_health.b")) { + t.Errorf("node %s writes the health reports %v", u.Node, perms.Publish) + } + if u.Kind == KindModule && u.Module == WatchmanModule && (!MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.b") || + MayPublish(perms, "$KV.mesh-host_health.b")) { + t.Errorf("the watchman reads the health reports %v, %v", MayPublish(perms, "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.b"), perms.Publish) + } if u.Kind == KindNode && (!MayPublish(perms, "$KV.mesh-host_packages.a") || MayPublish(perms, "$KV.mesh-host_packages.b")) { t.Errorf("node %s writes the packages keys %v", u.Node, perms.Publish) } diff --git a/go/buscompose/engine.go b/go/buscompose/engine.go index 4681729..bc504f3 100644 --- a/go/buscompose/engine.go +++ b/go/buscompose/engine.go @@ -58,6 +58,12 @@ func engineGrants(p Principal) (pub, sub []string) { if safeSubject.MatchString(p.Node) { pub = append(pub, readGrants(provisions.ProvisionsBucket, p.Node+".>")...) } + // And its node's own health report, its key `` alone (issue 774, the operator's alert table): what the watchman + // reads for the machine's alerts. + if safeSubject.MatchString(p.Node) { + pub = append(pub, append([]string{"$KV." + delivery.NodeHealthBucket + "." + p.Node}, + readGrants(delivery.NodeHealthBucket, p.Node)...)...) + } // And its node's declared packages, its own key alone (ADR 0565 clause 6). if safeSubject.MatchString(p.Node) { pub = append(pub, EnginePackagesGrants(p.Node)...) diff --git a/go/buscompose/writers.go b/go/buscompose/writers.go index 77abcd2..6da9693 100644 --- a/go/buscompose/writers.go +++ b/go/buscompose/writers.go @@ -248,6 +248,10 @@ var WritersTable = []WriterRow{ Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys}, // The acts a node's engine was asked through its node seat, and its own asks (novox/hq ADR 0563 §2, §3): the // node-engine's declared state, created by the grants part, written by each node's engine under its own keys. + // A node's own health report (issue 774): its engine's own key, read by the watchman. + {State: "a machine's health report", Writer: "the node-engine of the machine, its key `` alone", + KeptIn: "key-value " + delivery.NodeHealthBucket, Others: "read by the watchman through `reads`", + Subjects: kvOf(delivery.NodeHealthBucket), Writes: ownNodeKeys}, // A node's declared packages (novox/hq ADR 0565 clause 6): its engine's own key, read by pacman's verbs. {State: "a machine's declared packages", Writer: "the node-engine of the machine, its key `` alone", KeptIn: "key-value " + EnginePackagesBucket, Others: "read by pacman through `reads`", diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index 436867f..1d21efa 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -3,6 +3,7 @@ package delivery import ( "encoding/json" "testing" + "time" ) // ADR 0565 clauses 2 and 6 (issue 1953 L5-S2): the control records' layouts read what their writers write. The @@ -22,6 +23,14 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { {"a held value", `{"pull":"novox/mesh-catalog#42","head":"abc","declaration":"sha256:d","node":"novox","module":"mail",` + `"key":"quota","value":20,"by":"person.jochen","via":"telegram","why":"more room","approved":"the operator",` + `"proposal":"p-1","at":"2026-10-29T12:00:00Z"}`, &KeptValue{}}, + // A node's health report as mesh-host 65fbc1e6 encodes it (its test's look, struct order kept). + {"a node's health report", `{"v":1,"node":"anchor","network":{"state":"unhealthy","since":"2026-10-29T20:00:00Z",` + + `"parts":[{"part":"resolv-conf","state":"unhealthy","writer":"NetworkManager","owner":"dnsmasq","since":"2026-10-29T20:00:00Z"}]},` + + `"peers":[{"key":"A=","addresses":["10.10.0.2/32"],"seen":false},{"key":"B=","seen":true}],` + + `"units":{"state":"degraded","failed":[{"unit":"cups.service","scope":"system","since":"2026-10-29T20:00:00Z"}]},` + + `"relogin":[{"account":"jochen","module":"brother","resource":"brother.scanner","reason":"relogin needed: the session began before scanner","since":"2026-10-29T20:00:00Z"}],` + + `"found":[{"module":"photos","resource":"photos.library","path":"/srv/photos","reason":"used as found: owned by root","since":"2026-10-29T20:00:00Z"}],` + + `"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}}, {"the predefined list", `{"v":1,"commit":"abc","at":"2026-10-29T12:00:00Z","list":{"domain":{"type":"string"}}}`, &PredefinedKept{}}, } { if err := json.Unmarshal([]byte(c.raw), c.into); err != nil { @@ -31,10 +40,17 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { var want, got any _ = json.Unmarshal([]byte(c.raw), &want) _ = json.Unmarshal(again, &got) - if string(jsonOf(want)) != string(jsonOf(got)) || (c.name == "declared packages" && string(again) != c.raw) { + if string(jsonOf(want)) != string(jsonOf(got)) || ((c.name == "declared packages" || c.name == "a node's health report") && string(again) != c.raw) { t.Errorf("%s reads back as %s", c.name, again) } } + if a, b := (NodeHealth{Node: "anchor", At: time.Now()}), (NodeHealth{Node: "anchor"}); !SameNodeHealth(a, b) || + SameNodeHealth(a, NodeHealth{Node: "anchor", Peers: []PeerSeen{{Key: "A="}}}) { + t.Error("two reports differing only in when they were written are not the same, or differing ones are") + } + if _, err := EncodeNodeHealth(NodeHealth{}); err == nil { + t.Error("a report naming no node was encoded") + } if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil { t.Errorf("a held value's key is %q, %v", k, err) } diff --git a/go/delivery/nodehealth.go b/go/delivery/nodehealth.go new file mode 100644 index 0000000..f41486a --- /dev/null +++ b/go/delivery/nodehealth.go @@ -0,0 +1,146 @@ +package delivery + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "time" +) + +// **A node's own health report on the bus** (issue 774, the operator's alert table of 2026-10-29): one key per node, +// ``, of the node-engine's state `health` (NodeHealthBucket), written by that node's engine alone and only when +// what it says changed. The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the +// passing health message: machine-network (Network), network-rewritten (the resolver file's part, Writer and Owner), +// network-unreachable (Peers, read across every node's report), machine-units (Units), relogin-needed (Relogin) and +// directory-used-as-found (Found). Moved here from mesh-host internal/link (65fbc1e6) unchanged; the engine builds a +// report from its own health statement there (NodeHealthOf). + +// NodeHealthState and NodeHealthBucket are the node-engine's state health, as a reader names it and as the bus does. +const ( + NodeHealthState = "health" + NodeHealthBucket = "mesh-host_" + NodeHealthState + NodeHealthRead = "mesh-host." + NodeHealthState +) + +// NodeHealthVersion is the version of NodeHealth's layout. +const NodeHealthVersion = 1 + +// NodeHealth is one node's health report, as its engine last found it. +type NodeHealth struct { + V int `json:"v"` + Node string `json:"node"` + // Network is the machine's networking, its parts without their streaks; nil where the engine does not judge it yet. + Network *NetworkHealth `json:"network,omitempty"` + // Peers are the mesh tunnel's peers on the last look that read them, sorted by key. + Peers []PeerSeen `json:"peers,omitempty"` + // Units is the machine's service managers: each failed unit no module places. + Units *UnitsHealth `json:"units,omitempty"` + // Relogin are the accounts whose running session needs a new login for a group it was given. + Relogin []NodeAccount `json:"relogin,omitempty"` + // Found are the directories used as found. + Found []NodeDirectory `json:"found,omitempty"` + // At is when the engine wrote this, on a change: never a reason to write. + At time.Time `json:"at"` +} + +// NetworkHealth is the machine's networking (ADR 0241): the worst of its parts, since when, and each part. +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// NetworkPart is one part of a machine's networking, as its engine judged it on its second look. +type NetworkPart struct { + // Part is resolv-conf, names, tunnel, bus or route. + Part string `json:"part"` + State string `json:"state"` + // Reason is why it is not healthy, in words with no address, path or domain; Said the detail. + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + // Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose file it is. + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + // Toward is what the failure points at: "hub", or each resolver's address that failed. + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` +} + +// UnitsHealth is the machine's service managers (issue 315). +type UnitsHealth struct { + // State is running, degraded or unknown. + State string `json:"state"` + // Failed is every unit failed on two looks in a row that no module places. + Failed []FailedUnit `json:"failed"` + // Unread names a manager that could not be read, with why; Away an account's own manager not running. + Unread []string `json:"unread,omitempty"` + Away []string `json:"away,omitempty"` +} + +// FailedUnit is one failed unit no module places. +type FailedUnit struct { + Unit string `json:"unit"` + // Scope is system, or user: an account's own manager. + Scope string `json:"scope"` + Load string `json:"load,omitempty"` + Result string `json:"result,omitempty"` + Resource string `json:"resource,omitempty"` + Since time.Time `json:"since"` +} + +// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a +// handshake with it within five minutes. +type PeerSeen struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` +} + +// NodeAccount is one account marked relogin needed, with the module whose resource it is and why. +type NodeAccount struct { + Account string `json:"account"` + Module string `json:"module,omitempty"` + Resource string `json:"resource,omitempty"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why. +type NodeDirectory struct { + Module string `json:"module"` + Resource string `json:"resource"` + Path string `json:"path"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// EncodeNodeHealth is h as JSON at its version. +func EncodeNodeHealth(h NodeHealth) ([]byte, error) { + h.V = NodeHealthVersion + if h.Node == "" { + return nil, errors.New("a node's health report names its node") + } + return json.Marshal(h) +} + +// DecodeNodeHealth reads a report. +func DecodeNodeHealth(raw []byte) (NodeHealth, error) { + var h NodeHealth + if err := json.Unmarshal(raw, &h); err != nil { + return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err) + } + if h.V != NodeHealthVersion { + return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion) + } + return h, nil +} + +// SameNodeHealth says two reports say the same, whenever each was written. +func SameNodeHealth(a, b NodeHealth) bool { + a.At, b.At = time.Time{}, time.Time{} + ra, errA := json.Marshal(a) + rb, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(ra, rb) +} From 6eae0765f4d3eca42fd04d74a2dbed6e2f7e3fc9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:02:54 +0100 Subject: [PATCH 36/52] Name the machines a failing network part points at, so the watchman holds it there as the controller does without any machine's address (issue 774) NetworkPart.Targets: the hub's machine for "hub", the machine whose tunnel address is each resolver's; empty where the part is the machine's own. Filled by the node-engine from its tunnel's peers. Test: a row in TestTheControlRecordsReadTheirWritersValues, read back byte-equal. --- go/delivery/control_records_test.go | 7 ++++++- go/delivery/nodehealth.go | 12 +++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index 1d21efa..c43aeef 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -2,6 +2,7 @@ package delivery import ( "encoding/json" + "strings" "testing" "time" ) @@ -31,6 +32,10 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { `"relogin":[{"account":"jochen","module":"brother","resource":"brother.scanner","reason":"relogin needed: the session began before scanner","since":"2026-10-29T20:00:00Z"}],` + `"found":[{"module":"photos","resource":"photos.library","path":"/srv/photos","reason":"used as found: owned by root","since":"2026-10-29T20:00:00Z"}],` + `"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}}, + // A part pointing at another machine names it (issue 774): the hub's tunnel, held at the hub's machine. + {"a node's health report, a part pointing at the hub", `{"v":1,"node":"anchor","network":{"state":"unhealthy",` + + `"since":"2026-10-29T20:00:00Z","parts":[{"part":"tunnel","state":"unhealthy","reason":"the tunnel to the hub has not handshaken for over five minutes",` + + `"toward":["hub"],"targets":["novox"],"since":"2026-10-29T20:00:00Z"}]},"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}}, {"the predefined list", `{"v":1,"commit":"abc","at":"2026-10-29T12:00:00Z","list":{"domain":{"type":"string"}}}`, &PredefinedKept{}}, } { if err := json.Unmarshal([]byte(c.raw), c.into); err != nil { @@ -40,7 +45,7 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { var want, got any _ = json.Unmarshal([]byte(c.raw), &want) _ = json.Unmarshal(again, &got) - if string(jsonOf(want)) != string(jsonOf(got)) || ((c.name == "declared packages" || c.name == "a node's health report") && string(again) != c.raw) { + if string(jsonOf(want)) != string(jsonOf(got)) || ((c.name == "declared packages" || strings.HasPrefix(c.name, "a node's health report")) && string(again) != c.raw) { t.Errorf("%s reads back as %s", c.name, again) } } diff --git a/go/delivery/nodehealth.go b/go/delivery/nodehealth.go index f41486a..b9928a3 100644 --- a/go/delivery/nodehealth.go +++ b/go/delivery/nodehealth.go @@ -63,9 +63,15 @@ type NetworkPart struct { Writer string `json:"writer,omitempty"` Owner string `json:"owner,omitempty"` // Toward is what the failure points at: "hub", or each resolver's address that failed. - Toward []string `json:"toward,omitempty"` - Since time.Time `json:"since"` - Streak int `json:"streak,omitempty"` + Toward []string `json:"toward,omitempty"` + // Targets are the mesh machines the failure points at, by node name, as the controller holds a part at the machine + // it points at: the hub's machine for "hub", and the machine whose tunnel address is each resolver's. Empty where the + // part is the machine's own: it points at nothing, at this machine, or at an address no other mesh machine has (the + // resolver a VPN client wrote in). Filled by the node-engine from its tunnel's peers, so the watchman needs no + // address of any machine (issue 774). + Targets []string `json:"targets,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` } // UnitsHealth is the machine's service managers (issue 315). From 7fdbadc65516222ba72fb8985fbdcb13f4ecc61e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:09:25 +0100 Subject: [PATCH 37/52] Add the seat artifact-store, the seat-holders state with its reader, and the builder's log layout, so the store's sweep, who holds which seat and build history leave the controller (issue 1955, ADR 0571 clauses 1, 2, 4) --- go/buscompose/buscompose_test.go | 11 +- go/catalogue/artifact_store_seat.go | 52 ++++++++ go/catalogue/placement_seat.go | 5 +- go/catalogue/seats.go | 3 + go/catalogue/seats_test.go | 10 +- go/delivery/control_records_test.go | 38 ++++++ go/delivery/history_records.go | 179 ++++++++++++++++++++++++++++ 7 files changed, 291 insertions(+), 7 deletions(-) create mode 100644 go/catalogue/artifact_store_seat.go create mode 100644 go/delivery/history_records.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index c73690b..d02c957 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -611,6 +611,10 @@ func healthChecks(t *testing.T) { bucket("mesh-provisioner", "registry")}}, {Module: "mesh-vault", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}}, {Module: "postgres", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}}, + // ADR 0571 clauses 2, 4 (issue 1955): the artifact store's seat says artifacts-gone; the assigner keeps seat-holders. + {Module: "artifact-store-tools", Holds: []Seat{{Name: "artifact-store", Emits: []string{"artifacts-gone"}}}, + Reads: []string{delivery.SeatHoldersRead}}, + {Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}}, {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools", @@ -640,8 +644,11 @@ func healthChecks(t *testing.T) { // Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570 // clauses 1, 8). "a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"}, - "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, - "a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, + "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", + "mesh.seat.artifact-store.event.artifacts-gone"}, + // Only the assigner writes seat-holders. + "a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"}, + "a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, // Nor mesh-hand-acts' log and warrants, which it alone writes (ADR 0565 clause 10). "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.mesh-hand-acts_warrants.x", "mesh.log.mesh-hand-acts.acts.x"}, diff --git a/go/catalogue/artifact_store_seat.go b/go/catalogue/artifact_store_seat.go new file mode 100644 index 0000000..3fa1bf6 --- /dev/null +++ b/go/catalogue/artifact_store_seat.go @@ -0,0 +1,52 @@ +package catalogue + +// **The artifact store's sweep, out of the controller** (novox/hq ADR 0571 clause 2, issue 1955): the seat +// `artifact-store`, held once for the mesh by artifact-store-tools, serves what the controller served of its store +// (artifacts, held, mirrors, images, collect) and says `artifacts-gone`. Its keep set is the store's own listing joined +// with every want's fingerprint and each module's five newest builds from the builder's log (ADR 0526 §1); it lets go +// of nothing until the controller's lease names a holder that is no controller. + +// ArtifactStoreSeat is the seat artifact-store-tools holds. +const ArtifactStoreSeat = "artifact-store" + +// artifactStoreVerbs are the seat's verbs, with the controller's arguments so a caller changes only the seat it asks. +func artifactStoreVerbs() []Verb { + return []Verb{ + {Name: "artifacts", Description: "Every artifact the mesh made that lives in its artifact store, each with its " + + "repository, digest, kind (image or archive) and state: kept, and why (a want names it, or one of the five " + + "newest builds of its module in the builder's log), eligible (made, kept for no reason, not yet let go), or " + + "unrecorded (in the store and named by nothing the mesh made: never removed). Reads only.", + Input: schema(map[string]string{ + "repository": "only this repository's references, as / (the counts stay whole)", + "collected": "\"true\": list the collected references too (counted always)", + }, nil, "collected")}, + {Name: "held", Description: "Whether the artifact store holds each reference given, asked with one HEAD each: " + + "held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key. " + + "Reads only.", + Input: listed(schema(map[string]string{ + "references": "the artifacts' references, artifact-store:////blobs/sha256:: a list, " + + "or one text separated by commas", + }, []string{"references"}), "references")}, + {Name: "collect", Description: "The artifact store's sweep, now: what is eligible is let go of. Without confirm a " + + "dry run: what would be let go, nothing held or deleted. With confirm and why: every kept archive held first, " + + "then each eligible artifact let go of, oldest first, an index with the platform manifests no kept index " + + "names, each deletion recorded in the hand-act log. Bounded by most (500 by default, at most 5000) and 45 " + + "seconds; what is left is said. Never touches a digest the mesh did not make.", + Input: schema(map[string]string{ + "confirm": "\"true\": let go of what is eligible (needs why); without it nothing is done", + "why": "with confirm: why, required and recorded", + "most": "let go of at most this many (500 by default, at most 5000)", + }, nil, "confirm")}, + {Name: "mirrors", Description: "The bases the mesh copied into its artifact store for builds to stand on, each " + + "with its state: kept because a kept build stood on it, or eligible. With record and confirm and why, copies " + + "no record names are recorded as the mesh's. Deletes nothing.", + Input: schema(map[string]string{ + "record": "references to record, artifact-store://@sha256:, separated by spaces or commas", + "confirm": "\"true\": record them (needs why); without it nothing is recorded", + "why": "with confirm: why, required and recorded", + }, nil, "confirm")}, + {Name: "images", Description: "Every container image one machine's want and have name, each with the resources " + + "naming it: what the machine keeps when it prunes its images. Reads only.", + Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})}, + } +} diff --git a/go/catalogue/placement_seat.go b/go/catalogue/placement_seat.go index c400d08..7fa69de 100644 --- a/go/catalogue/placement_seat.go +++ b/go/catalogue/placement_seat.go @@ -39,7 +39,10 @@ func placementVerbs() []Verb { "history, one node's or one module's changes in order: a revision the state keeps whole, an older one as the " + "proposal's changes from its placement-changed event.", Input: schema(map[string]string{"node": "one node", "module": "one module: the nodes it is placed on", - "history": "\"true\": the changes in order, from the stream and the state's revisions"}, nil, "history")}, + "history": "\"true\": the changes in order, from the stream and the state's revisions", + // ADR 0571 clause 4: who holds which seat on which node, from the state seat-holders. + "seats": "\"true\": who holds each seat, and on which node (the state seat-holders); with node, that node's"}, + nil, "history", "seats")}, {Name: PlacementVerbProposals, Optional: true, Description: "Every open proposal of placement: its changes, " + "the revision of each key it was laid over, its candidates, every node's answer, the differences computed and " + "answered, its verdict and its ask. Narrowed to one by id.", diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 7f39dcf..b45ff1e 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -303,6 +303,9 @@ var defaultSeats = append([]Seat{ // policy and the bus step at the assigner, and the hand-act log. {Name: WantsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"policy-changed"}, Serves: wantsVerbs()}, {Name: HandActsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"act-recorded"}, Serves: handActsVerbs()}, + // The artifact store's sweep (novox/hq ADR 0571 clause 2, artifact_store_seat.go). + {Name: ArtifactStoreSeat, Scope: ScopeMesh, Decision: "novox/hq ADR 0571", Emits: []string{"artifacts-gone"}, + Serves: artifactStoreVerbs()}, // Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a // delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight. // They keep their names until that migration is done deliberately, apart from the node-* pass. diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index 828464e..3182411 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -76,9 +76,11 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // And ADR 0565 clause 1's two. {WantsSeat, "upgrade,bus", "policy-changed"}, {HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"}, + // ADR 0571 clause 2: the artifact store's sweep. + {ArtifactStoreSeat, "artifacts,held,collect,mirrors,images", "artifacts-gone"}, } { s, ok := SeatNamed(c.seat) - if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 056") || + if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 05") || strings.Join(VerbNames(s.Serves), ",") != c.verbs || strings.Join(s.Emits, ",") != c.emits { t.Errorf("the health seat %s is %v, %+v", c.seat, ok, s) } @@ -120,9 +122,9 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // mesh-build-machine row goes, when no registered manifest claims it. // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1); // fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts - // (novox/hq ADR 0565 clause 1). - if len(Seats()) != 55 { - t.Errorf("the mesh defines %d seats rather than 55; the set is closed, so a change here is "+ + // (novox/hq ADR 0565 clause 1); fifty-six with artifact-store (novox/hq ADR 0571 clause 2). + if len(Seats()) != 56 { + t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames()) } } diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index c43aeef..b41631c 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -56,6 +56,44 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { if _, err := EncodeNodeHealth(NodeHealth{}); err == nil { t.Error("a report naming no node was encoded") } + // ADR 0571 clause 4: who holds which seat, from placement; a node seat by its node. + held := SeatHoldersOf(map[string]Placement{ + "ace": {Node: "ace", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}}, + "novox": {Node: "novox", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"}}}, + }) + keys := map[string][]byte{} + for k, v := range held { + keys[k], _ = json.Marshal(v) + } + if h, err := HoldersOf(keys, "mesh-dns-resolver", ""); err != nil || len(h) != 2 || h[0].Node != "ace" { + t.Errorf("a replicated seat's holders are %v, %v", h, err) + } + if h, _ := HoldersOf(keys, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 3 { + t.Errorf("a node seat's holder is %v, keys %v", h, held) + } + if h, _ := HoldersOf(keys, "node-files", "novox"); h != nil { + t.Errorf("a node seat nobody holds there is held by %v", h) + } + // ADR 0571 clause 1: the builder's log, a start and an end, and the duration between them. + at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Node: "ace", At: at} + end := start + end.Event, end.At, end.Outcome, end.Artifacts = BuildEndedEvent, at.Add(3*time.Minute), "done", []BuiltDigest{{Name: "image", Digest: "sha256:1"}} + for _, e := range []BuildLogEntry{start, end} { + raw, err := EncodeBuildLogEntry(e) + back, derr := DecodeBuildLogEntry(raw) + if err != nil || derr != nil || back.Event != e.Event || back.Outcome != e.Outcome { + t.Errorf("%s read back as %+v, %v %v", e.Event, back, err, derr) + } + } + if d, ok := BuildDuration(start, end); !ok || d != 3*time.Minute { + t.Errorf("the build took %s, %v", d, ok) + } + bad := end + bad.Outcome = "" + if _, err := EncodeBuildLogEntry(bad); err == nil { + t.Error("an end with no outcome was encoded") + } if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil { t.Errorf("a held value's key is %q, %v", k, err) } diff --git a/go/delivery/history_records.go b/go/delivery/history_records.go new file mode 100644 index 0000000..0b49576 --- /dev/null +++ b/go/delivery/history_records.go @@ -0,0 +1,179 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "sort" + "time" +) + +// **Who holds which seat, and the builder's log** (novox/hq ADR 0571 clauses 1 and 4, issue 1955). + +// SeatHoldersState is the state the seat mesh-placement's holder writes of who holds which seat on which node (clause 4): +// one key per seat held once for the mesh, ``, and one per node of a seat held on every node, `.`. +const ( + SeatHoldersState = "seat-holders" + SeatHoldersBucket = AssignerModule + "_" + SeatHoldersState + SeatHoldersRead = AssignerModule + "." + SeatHoldersState +) + +// SeatHolder is one module holding a seat on a node. +type SeatHolder struct { + Module string `json:"module"` + Node string `json:"node"` +} + +// SeatHolders is one key of seat-holders: the seat, its scope, the node for a node seat's key, and its holders +// (several only for a replicated mesh seat), sorted. +type SeatHolders struct { + V int `json:"v"` + Seat string `json:"seat"` + Scope string `json:"scope"` + Node string `json:"node,omitempty"` + Holders []SeatHolder `json:"holders"` +} + +// SeatHoldersKey is a seat's key: `` for a seat held once for the mesh (node empty), `.` for a node +// seat on one node. +func SeatHoldersKey(seat, node string) (string, error) { + if err := checkName("seat", seat); err != nil { + return "", err + } + if node == "" { + return seat, nil + } + if err := checkName("node", node); err != nil { + return "", err + } + return seat + "." + node, nil +} + +// SeatHoldersOf is the state composed from every node's placement (ADR 0567 §1): each holding by the seat it claims, +// keyed as SeatHoldersKey says, with the module holding it and its node. Deterministic, so a writer writes only a key +// that differs. +func SeatHoldersOf(placements map[string]Placement) map[string]SeatHolders { + out := map[string]SeatHolders{} + nodes := make([]string, 0, len(placements)) + for n := range placements { + nodes = append(nodes, n) + } + sort.Strings(nodes) + for _, n := range nodes { + for _, h := range placements[n].Holds { + node := "" + if h.Scope == "node" { + node = n + } + key, err := SeatHoldersKey(h.Claim, node) + if err != nil { + continue + } + sh := out[key] + sh.V, sh.Seat, sh.Scope, sh.Node = Version, h.Claim, h.Scope, node + sh.Holders = append(sh.Holders, SeatHolder{Module: h.Module, Node: n}) + out[key] = sh + } + } + for k, sh := range out { + sort.Slice(sh.Holders, func(i, j int) bool { + return sh.Holders[i].Node+"/"+sh.Holders[i].Module < sh.Holders[j].Node+"/"+sh.Holders[j].Module + }) + out[k] = sh + } + return out +} + +// HoldersOf reads who holds seat (on node, for a node seat) from the state's keys as a reader watched them: the node's +// key first, then the seat's. None when nobody holds it. +func HoldersOf(keys map[string][]byte, seat, node string) ([]SeatHolder, error) { + for _, n := range []string{node, ""} { + key, err := SeatHoldersKey(seat, n) + if err != nil { + if n == "" { + return nil, err + } + continue + } + raw, ok := keys[key] + if !ok { + continue + } + var sh SeatHolders + if err := json.Unmarshal(raw, &sh); err != nil { + return nil, fmt.Errorf("delivery: the holders of %s are not readable: %w", key, err) + } + if sh.V != Version { + return nil, fmt.Errorf("delivery: the holders of %s are of version %d, and this reads %d", key, sh.V, Version) + } + return sh.Holders, nil + } + return nil, nil +} + +// The builder's log (clause 1): a stream of its own, `log`, one entry when a build of a fingerprint starts and one when +// it ends, kept 30 days. +const ( + BuilderLogStream = "log" + BuildStartedEvent = "build.started" + BuildEndedEvent = "build.ended" + // BuilderLogDays is how long the log keeps its entries. + BuilderLogDays = 30 +) + +// BuiltDigest is one artifact a build ended with, by its name and digest. +type BuiltDigest struct { + Name string `json:"name"` + Digest string `json:"digest"` +} + +// BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, the build node and +// the time; at the end, the outcome and each artifact's digest. +type BuildLogEntry struct { + V int `json:"v"` + Event string `json:"event"` + Module string `json:"module"` + FP string `json:"fp"` + Source string `json:"source"` + Node string `json:"node"` + At time.Time `json:"at"` + Outcome string `json:"outcome,omitempty"` + Artifacts []BuiltDigest `json:"artifacts,omitempty"` +} + +// EncodeBuildLogEntry is e as JSON at Version; it refuses an entry a reader could not place. +func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) { + e.V = Version + switch { + case e.Event != BuildStartedEvent && e.Event != BuildEndedEvent: + return nil, fmt.Errorf("delivery: a build log entry is %s or %s, not %q", BuildStartedEvent, BuildEndedEvent, e.Event) + case e.Module == "" || e.FP == "" || e.Node == "" || e.At.IsZero(): + return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, node and time") + case e.Event == BuildEndedEvent && e.Outcome == "": + return nil, fmt.Errorf("delivery: a build's end names its outcome") + case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Artifacts) > 0): + return nil, fmt.Errorf("delivery: a build's start names no outcome and no artifact") + } + return json.Marshal(e) +} + +// DecodeBuildLogEntry reads one entry. +func DecodeBuildLogEntry(data []byte) (BuildLogEntry, error) { + var e BuildLogEntry + if err := json.Unmarshal(data, &e); err != nil { + return BuildLogEntry{}, fmt.Errorf("delivery: a build log entry is not readable: %w", err) + } + if e.V != Version { + return BuildLogEntry{}, fmt.Errorf("delivery: a build log entry of version %d, and this reads %d", e.V, Version) + } + return e, nil +} + +// BuildDuration is a build's duration, its end less its start (clause 1): false unless both are the same module's +// build of the same fingerprint, the start first. +func BuildDuration(start, end BuildLogEntry) (time.Duration, bool) { + if start.Event != BuildStartedEvent || end.Event != BuildEndedEvent || start.Module != end.Module || start.FP != end.FP || + end.At.Before(start.At) { + return 0, false + } + return end.At.Sub(start.At), true +} From f9a496d3e12151fd55f0fcfc5ecba47ba8644bd1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:09:47 +0100 Subject: [PATCH 38/52] Carry each build's repository, directory and commit in the builder's log, so the indexer, the outside-built registrations and the assigner read where a build came from there (issue 1955) --- go/delivery/control_records_test.go | 16 +++++++++------ go/delivery/history_records.go | 31 +++++++++++++++++------------ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index b41631c..94683f4 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -76,23 +76,27 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { } // ADR 0571 clause 1: the builder's log, a start and an end, and the duration between them. at := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) - start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Node: "ace", At: at} + start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Repository: "novox/mesh-catalog", + Directory: "modules/letta", Commit: "abc123", Node: "ace", At: at} end := start end.Event, end.At, end.Outcome, end.Artifacts = BuildEndedEvent, at.Add(3*time.Minute), "done", []BuiltDigest{{Name: "image", Digest: "sha256:1"}} for _, e := range []BuildLogEntry{start, end} { raw, err := EncodeBuildLogEntry(e) back, derr := DecodeBuildLogEntry(raw) - if err != nil || derr != nil || back.Event != e.Event || back.Outcome != e.Outcome { + if err != nil || derr != nil || back.Event != e.Event || back.Outcome != e.Outcome || back.Directory != "modules/letta" || + back.Repository != "novox/mesh-catalog" || back.Commit != "abc123" { t.Errorf("%s read back as %+v, %v %v", e.Event, back, err, derr) } } if d, ok := BuildDuration(start, end); !ok || d != 3*time.Minute { t.Errorf("the build took %s, %v", d, ok) } - bad := end - bad.Outcome = "" - if _, err := EncodeBuildLogEntry(bad); err == nil { - t.Error("an end with no outcome was encoded") + for _, edit := range []func(*BuildLogEntry){func(e *BuildLogEntry) { e.Outcome = "" }, func(e *BuildLogEntry) { e.Commit = "" }} { + bad := end + edit(&bad) + if _, err := EncodeBuildLogEntry(bad); err == nil { + t.Errorf("%+v was encoded", bad) + } } if k, err := HeldValueKey("novox/mesh-catalog#42", "novox", "mail", "quota"); k != "novox.mesh-catalog.42.novox.mail.quota" || err != nil { t.Errorf("a held value's key is %q, %v", k, err) diff --git a/go/delivery/history_records.go b/go/delivery/history_records.go index 0b49576..6284549 100644 --- a/go/delivery/history_records.go +++ b/go/delivery/history_records.go @@ -126,18 +126,23 @@ type BuiltDigest struct { Digest string `json:"digest"` } -// BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, the build node and -// the time; at the end, the outcome and each artifact's digest. +// BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, where the module is +// built from (its repository, its directory there and the commit), the build node and the time; at the end, the outcome +// and each artifact's digest. The repository, directory and commit are what the indexer's PathOf, the outside-built +// registrations and the assigner's running commits read (the lead, 2026-10-29). type BuildLogEntry struct { - V int `json:"v"` - Event string `json:"event"` - Module string `json:"module"` - FP string `json:"fp"` - Source string `json:"source"` - Node string `json:"node"` - At time.Time `json:"at"` - Outcome string `json:"outcome,omitempty"` - Artifacts []BuiltDigest `json:"artifacts,omitempty"` + V int `json:"v"` + Event string `json:"event"` + Module string `json:"module"` + FP string `json:"fp"` + Source string `json:"source"` + Repository string `json:"repository"` + Directory string `json:"directory"` + Commit string `json:"commit"` + Node string `json:"node"` + At time.Time `json:"at"` + Outcome string `json:"outcome,omitempty"` + Artifacts []BuiltDigest `json:"artifacts,omitempty"` } // EncodeBuildLogEntry is e as JSON at Version; it refuses an entry a reader could not place. @@ -146,8 +151,8 @@ func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) { switch { case e.Event != BuildStartedEvent && e.Event != BuildEndedEvent: return nil, fmt.Errorf("delivery: a build log entry is %s or %s, not %q", BuildStartedEvent, BuildEndedEvent, e.Event) - case e.Module == "" || e.FP == "" || e.Node == "" || e.At.IsZero(): - return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, node and time") + case e.Module == "" || e.FP == "" || e.Node == "" || e.At.IsZero() || e.Repository == "" || e.Commit == "": + return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, repository, commit, node and time") case e.Event == BuildEndedEvent && e.Outcome == "": return nil, fmt.Errorf("delivery: a build's end names its outcome") case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Artifacts) > 0): From dc78524e38659a278d4a6e37ab45b043e4ba17be Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:10:46 +0100 Subject: [PATCH 39/52] Let the artifact store's seat holder read the controller's lease, so its sweep starts only once the lease names no controller (issue 1955, ADR 0571 clause 2) --- go/buscompose/buscompose_test.go | 2 +- go/buscompose/userlist_switch.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index d02c957..0f6b29f 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -585,7 +585,7 @@ func healthChecks(t *testing.T) { may bool }{ {"mesh-conditions", ControllerLease, true}, {"mesh-healers", ControllerLease, true}, {WatchmanModule, ControllerLease, true}, - {"mesh-self-check", ControllerLease, false}, {"messenger", ControllerLease, false}, + {"mesh-self-check", ControllerLease, false}, {"messenger", ControllerLease, false}, {ArtifactStoreTools, ControllerLease, true}, {"mesh-conditions", ControllerConditions, true}, {"mesh-healers", ControllerConditions, false}, {"messenger", ControllerConditions, false}, {"mesh-healers", ControllerHandActs, true}, {"mesh-conditions", ControllerHandActs, false}, // ADR 0565 clauses 10, 11 (issue 1953 L5-S4): mesh-hand-acts reads the controller's log beside its own. diff --git a/go/buscompose/userlist_switch.go b/go/buscompose/userlist_switch.go index bae9341..1bc1303 100644 --- a/go/buscompose/userlist_switch.go +++ b/go/buscompose/userlist_switch.go @@ -79,6 +79,9 @@ func grantsOwnObjects(p Principal, pattern string) bool { // readers too (announce.State.Readers); both are honoured. const VaultBusAccounts = "mesh-vault.bus-accounts" +// ArtifactStoreTools is the module holding the seat artifact-store (novox/hq ADR 0571 clause 2). +const ArtifactStoreTools = "artifact-store-tools" + // NamedReaders are the states only the modules named may read, by `.`: the grants part's wanted accounts // are read by the vault alone (novox/hq ADR 0561 §2). // The controller's lease is read by the parts that turn on at its write (ADR 0561 §3, §6), and by the assigner, which @@ -88,7 +91,9 @@ const VaultBusAccounts = "mesh-vault.bus-accounts" // watchman read the lease; the keeper alone the controller's conditions; mesh-healers and mesh-hand-acts its hand-act log // (ADR 0565 clauses 10, 11: every reader of acts reads both logs); and nobody the healers' `would` (health_grants.go). var NamedReaders = map[string][]string{VaultBusAccounts: {GrantsModule}, GrantsAccounts: {VaultSeat}, - ControllerLease: {VaultSeat, GrantsModule, delivery.AssignerModule, health.KeeperModule, health.HealersModule, WatchmanModule}, + ControllerLease: {VaultSeat, GrantsModule, delivery.AssignerModule, health.KeeperModule, health.HealersModule, WatchmanModule, + // ADR 0571 clause 2: the artifact store's seat sweeps only once the lease names no controller. + ArtifactStoreTools}, ControllerConditions: {health.KeeperModule}, ControllerHandActs: {health.HealersModule, handacts.Module}, HealersWould: {}} // ReaderMay says whether module may read the state read (`.`) by the named readers: always, for a state From 196c5b6af277e9b398f83a1d4adee3c113e9ff3d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:16:56 +0100 Subject: [PATCH 40/52] Compose a node's placed data items from the bus and move the readings layout into the sdk, so disk-load measures without a composed file and restic, mounts, nfs-server and the self-check read one layout (issue 1955, ADR 0571 clause 3) --- go/buscompose/buscompose_test.go | 9 +- go/catalogue/data_placed.go | 98 ++++++++++++++++++++++ go/catalogue/data_test.go | 10 +++ go/compose/frombus/frombus.go | 10 +++ go/compose/frombus/frombus_test.go | 12 +++ go/compose/node.go | 16 ++++ go/health/condition_test.go | 27 ++++++ go/health/readings.go | 127 +++++++++++++++++++++++++++++ 8 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 go/catalogue/data_placed.go create mode 100644 go/health/readings.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 0f6b29f..5e356b4 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -615,6 +615,9 @@ func healthChecks(t *testing.T) { {Module: "artifact-store-tools", Holds: []Seat{{Name: "artifact-store", Emits: []string{"artifacts-gone"}}}, Reads: []string{delivery.SeatHoldersRead}}, {Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}}, + // ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them. + {Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}}, + {Module: "restic", Reads: []string{"disk-load.readings"}}, {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools", @@ -630,6 +633,8 @@ func healthChecks(t *testing.T) { "a.mesh-vault": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, + "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, + "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data"}, WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.INFO.SEAT_TELEGRAM_SENDER", @@ -648,7 +653,9 @@ func healthChecks(t *testing.T) { "mesh.seat.artifact-store.event.artifacts-gone"}, // Only the assigner writes seat-holders. "a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"}, - "a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, + // Only disk-load writes its readings. + "a.restic": {"$KV.disk-load_readings.a.pg.data"}, + "a.postgres": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak"}, // Nor mesh-hand-acts' log and warrants, which it alone writes (ADR 0565 clause 10). "a.mesh-healers": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.mesh-hand-acts_warrants.x", "mesh.log.mesh-hand-acts.acts.x"}, diff --git a/go/catalogue/data_placed.go b/go/catalogue/data_placed.go new file mode 100644 index 0000000..0744fe6 --- /dev/null +++ b/go/catalogue/data_placed.go @@ -0,0 +1,98 @@ +package catalogue + +import ( + "fmt" + "regexp" + "strings" +) + +// **The data items of a node's modules, placed, read from the composition** (novox/hq ADR 0571 clause 3, ADR 0233 §22 to +// §25): disk-load measures each item every module on its node declares, and the node-backup holder, mounts and nfs-server +// read the readings. Each item is its module's declaration with its path resolved to this node's place, its protection +// and how it is measured: the same lines the node-backup holder is handed as `${backup:data}`, composed by the same +// function, so a reader that composes from the bus and a holder handed the file read one list. + +// PlacedDataItem is one data item a module on a node declares, as placed on that node. +type PlacedDataItem struct { + Module string `json:"module"` + ID string `json:"item"` + Class string `json:"class"` + // Path is the item's place on the node; CoveredBy the path whose snapshot keeps it (the item, or the item its dump + // writes into), empty when it is not backed up. + Path string `json:"path"` + CoveredBy string `json:"covered_by,omitempty"` + // Protection is backup, redundancy, both or none; Measure walk, dataset or shallow. + Protection string `json:"protection,omitempty"` + Measure string `json:"measure,omitempty"` +} + +var ( + dataModuleHeader = regexp.MustCompile(`^#\s*([a-z0-9][a-z0-9-]*)$`) + dataSafePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`) +) + +// ParseDataItems reads the node-backup holder's data text (`${backup:data}`: a `# ` line, then one line per item, +// `item `) into each module's items, in order. A line it does +// not read is refused, naming the module: a path with a space or a quote is never handed to a measuring shell. +func ParseDataItems(text string) (map[string][]PlacedDataItem, error) { + out := map[string][]PlacedDataItem{} + module := "" + for _, raw := range strings.Split(text, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if m := dataModuleHeader.FindStringSubmatch(line); m != nil { + module = m[1] + continue + } + if strings.HasPrefix(line, "#") || module == "" { + continue + } + f := strings.Fields(line) + if len(f) < 5 || len(f) > 7 || f[0] != "item" || !dataSafePath.MatchString(f[3]) || (f[4] != "-" && !dataSafePath.MatchString(f[4])) { + return nil, fmt.Errorf("%s declares a data line that is not read: %s", module, line) + } + it := PlacedDataItem{Module: module, ID: f[1], Class: f[2], Path: f[3], Measure: MeasureWalk} + if len(f) >= 6 { + it.Protection = f[5] + } + if len(f) == 7 { + it.Measure = f[6] + } + if it.Measure != MeasureWalk && it.Measure != MeasureDataset && it.Measure != MeasureShallow { + return nil, fmt.Errorf("%s declares a data line that is not read: %s", module, line) + } + if f[4] != "-" { + it.CoveredBy = f[4] + } + out[module] = append(out[module], it) + } + return out, nil +} + +// DataItems is every data item the resolution's modules declare, placed on its node with the rendering given: the text +// the node-backup holder is handed, read. A module left out of the node's declaration keeps its items, as its data is +// still on the node (ADR 0262); an item that cannot be placed is said in unplaced and left out. +func (r Resolution) DataItems(with Rendering) (items map[string][]PlacedDataItem, unplaced []string, err error) { + left := r.LeftOut(with.Settings, with.Adopted) + for module, why := range with.NotPlaced { + if _, already := left[module]; !already { + left[module] = why + } + } + modules := make([]Manifest, 0, len(r.Modules)) + for _, m := range r.Modules { + if _, isLeft := left[m.Module]; isLeft { + m = backupView(m) + } + modules = append(modules, m) + } + text, unplaced, err := seatContributions(modules, Manifest{}, BackupSeat+":"+BackupKindData, with, + machineFacts(r, with.Names, with.MeshRange), r.Capabilities) + if err != nil { + return nil, unplaced, err + } + items, err = ParseDataItems(text) + return items, unplaced, err +} diff --git a/go/catalogue/data_test.go b/go/catalogue/data_test.go index 9511cda..a013e9c 100644 --- a/go/catalogue/data_test.go +++ b/go/catalogue/data_test.go @@ -192,6 +192,16 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { if data != want { t.Fatalf("data lines:\n%s\nwant:\n%s", data, want) } + // ADR 0571 clause 3: the same text, read into items, as disk-load and the backup holder read it; a path a shell + // would split is refused. + items, err := ParseDataItems(data) + if err != nil || len(items) != 3 || items["media"][0] != (PlacedDataItem{Module: "media", ID: "films", Class: "irreplaceable", + Path: "/tank/films", Protection: "redundancy", Measure: MeasureDataset}) || items["pg"][0].CoveredBy != "/srv/store/dumps" { + t.Fatalf("the data lines read as %+v: %v", items, err) + } + if _, err := ParseDataItems("# a\nitem x cache /a b - none walk\n"); err == nil { + t.Fatal("a path with a space was read") + } if _, _, err := seatContributions([]Manifest{agent}, Manifest{}, BackupSeat+":"+BackupKindData, Rendering{}, nil, nil); err == nil { t.Fatal("a home directory with no account on the machine reached the holder as a placeholder") } diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 8478333..2fedc4c 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -853,3 +853,13 @@ func membershipOf(own delivery.Composed) string { } return "" } + +// DataItems is every data item the modules placed on scope's node declare, placed there, composed from the bus as Plans +// composes (novox/hq ADR 0571 clause 3): disk-load measures these. +func DataItems(ctx context.Context, r Reader, mesh Mesh, s Scope) (map[string][]catalogue.PlacedDataItem, []string, error) { + in, err := Input(ctx, r, mesh, s) + if err != nil { + return nil, nil, err + } + return compose.DataItems(in, s.Node) +} diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 69d6333..1a4b562 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -712,6 +712,18 @@ func TestInput(t *testing.T) { wj, _ := json.MarshalIndent(controllers, "", " ") t.Errorf("the seat's plans differ from the controller's:\n got %s\nwant %s", gj, wj) } + // ADR 0571 clause 3: the data items of n1's modules, placed, from the bus alone: what disk-load measures. + items, unplaced, err := DataItems(ctx, noSecrets, Mesh{}, Scope{Node: "n1"}) + if err != nil || len(unplaced) > 0 || len(items) == 0 { + t.Errorf("the data items from the bus are %v, unplaced %v: %v", items, unplaced, err) + } + for module, its := range items { + for _, it := range its { + if it.Module != module || !strings.HasPrefix(it.Path, "/") || it.Measure == "" { + t.Errorf("%s's item %+v is not placed", module, it) + } + } + } // And what a provider on another node serves from its settings (mailu's smtp domain, issue 1126: shanks refused // '${setting:domain} ... nothing sets "domain"'): read for that provider's served keys alone, through Served. diff --git a/go/compose/node.go b/go/compose/node.go index 858f77d..81e8557 100644 --- a/go/compose/node.go +++ b/go/compose/node.go @@ -333,3 +333,19 @@ func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string sort.Strings(out) return out } + +// DataItems is every data item the modules placed on a node declare, placed there (novox/hq ADR 0571 clause 3): what the +// node-backup holder is handed as `${backup:data}`, composed over the Input as Compose composes the node, so disk-load +// reads the list from the bus with no composed file. Unplaced names each item left out and why. +func DataItems(in Input, node string) (map[string][]catalogue.PlacedDataItem, []string, error) { + c := newComposer(in) + resolved, err := c.planFor(node) + if err != nil { + return nil, nil, err + } + rendered, err := c.render(node, Planning) + if err != nil { + return nil, nil, err + } + return resolved.Plan.DataItems(rendered.Rendering) +} diff --git a/go/health/condition_test.go b/go/health/condition_test.go index e62ea04..9c25aab 100644 --- a/go/health/condition_test.go +++ b/go/health/condition_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the @@ -65,6 +67,31 @@ func TestTheControllersConditionEventsReadUnchanged(t *testing.T) { // take-over key the readers follow. func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + // ADR 0571 clause 3: a reading keeps its seven-day peak and is written only when it changes; a partial size is no peak. + var prev *Reading + for i, step := range []struct { + day, size int64 + partial, wrote bool + peak int64 + }{{20, 100, false, true, 100}, {20, 100, false, false, 100}, {21, 300, false, true, 300}, {22, 50, false, true, 300}, + {23, 9000, true, true, 300}, {29, 10, false, true, 50}, {30, 10, false, true, 10}} { + now := time.Date(2026, 10, int(step.day), 1, 0, 0, 0, time.UTC) + size := step.size + m := Measured{SizeBytes: &size, MeasuredAt: &now, Precision: "exact"} + if step.partial { + m.Precision = "partial: stopped" + } + r, wrote := NextReading(prev, "ace", catalogue.PlacedDataItem{Module: "pg", ID: "data", Path: "/srv/pg"}, m, now) + raw, _ := json.Marshal(r) + back, err := DecodeReading(raw) + if wrote != step.wrote || r.Peak == nil || *r.Peak != step.peak || err != nil || back.ID != "data" { + t.Fatalf("reading %d: wrote %v, peak %v, read back %v", i, wrote, r.Peak, err) + } + prev = &r + } + if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" { + t.Fatal("the readings' key or bucket moved") + } good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing", Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} diff --git a/go/health/readings.go b/go/health/readings.go new file mode 100644 index 0000000..a0983c9 --- /dev/null +++ b/go/health/readings.go @@ -0,0 +1,127 @@ +package health + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// **Disk usage readings and peaks per data item** (novox/hq ADR 0571 clause 3, ADR 0233 §21, §27, §34): disk-load, on every +// node, measures each data item the modules of its node declare (catalogue.PlacedDataItem, from compose/frombus.DataItems) +// and keeps the newest reading and the item's largest size in seven days as its own state `readings`, one key +// `..`, written only when it changes. The node-backup holder, mounts, nfs-server and the self-check's +// probes read it there; the controller keeps none of it. Moved here from mesh-catalog modules/disk-load (b4c237d2). +const ( + DiskLoadModule = "disk-load" + ReadingsState = "readings" + ReadingsBucket = DiskLoadModule + "_" + ReadingsState + ReadingsRead = DiskLoadModule + "." + ReadingsState +) + +// ReadingVersion is the version of a reading's layout. +const ReadingVersion = 1 + +// PeakWindow is how far back the peak looks (ADR 0233 §34). +const PeakWindow = 7 * 24 * time.Hour + +// Redundancy is the redundant storage an item is on, as read: zfs, md or btrfs, its pool, array or filesystem, and its +// health (nil when it could not be read). +type Redundancy struct { + Kind string `json:"kind"` + Where string `json:"where"` + Healthy *bool `json:"healthy"` + Said string `json:"said"` +} + +// Measured is what one measurement found. Precision says what the size is: "exact", "dataset : its whole size", +// "partial: ..." (a lower bound, never compared), or "no size: ...". +type Measured struct { + SizeBytes *int64 `json:"size_bytes,omitempty"` + LastWrite *time.Time `json:"last_write,omitempty"` + MeasuredAt *time.Time `json:"measured_at,omitempty"` + Precision string `json:"precision,omitempty"` + Error string `json:"error,omitempty"` + Redundancy *Redundancy `json:"redundancy,omitempty"` +} + +// Partial says the size is only a lower bound, which a peak is never made of. +func (m Measured) Partial() bool { return strings.HasPrefix(m.Precision, "partial") } + +// DaySize is the largest size measured on one UTC day. +type DaySize struct { + Day string `json:"day"` + Bytes int64 `json:"bytes"` +} + +// Reading is the value of one key: the item as placed, what was measured, and the largest size in the seven days up to +// it (Peak, measured on PeakAt), read from the day maxima Days. +type Reading struct { + V int `json:"v"` + Node string `json:"node"` + catalogue.PlacedDataItem + Measured + Peak *int64 `json:"peak_bytes,omitempty"` + PeakAt string `json:"peak_day,omitempty"` + Days []DaySize `json:"days,omitempty"` +} + +// ReadingKey is the key of an item on a node: `..`. +func ReadingKey(node, module, item string) string { return node + "." + module + "." + item } + +// DecodeReading reads a key's value; it refuses another version. +func DecodeReading(raw []byte) (Reading, error) { + var r Reading + if err := json.Unmarshal(raw, &r); err != nil { + return Reading{}, fmt.Errorf("health: a reading is not readable: %w", err) + } + if r.V != ReadingVersion { + return Reading{}, fmt.Errorf("health: a reading of version %d, and this reads %d", r.V, ReadingVersion) + } + return r, nil +} + +// NextReading is the reading after a measurement, and whether it differs from the one before in anything but the time it +// was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries +// the last size's days on, so the peak outlives one failed measurement. +func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) { + r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m} + var days []DaySize + if prev != nil { + days = prev.Days + } + today := now.UTC().Format("2006-01-02") + cut := now.UTC().Add(-PeakWindow).Format("2006-01-02") + var kept []DaySize + todays := int64(-1) + for _, d := range days { + switch { + case d.Day == today: + todays = d.Bytes + case d.Day >= cut: + kept = append(kept, d) + } + } + if m.SizeBytes != nil && !m.Partial() && *m.SizeBytes > todays { + todays = *m.SizeBytes + } + if todays >= 0 { + kept = append(kept, DaySize{Day: today, Bytes: todays}) + } + r.Days = kept + for _, d := range kept { + if r.Peak == nil || d.Bytes >= *r.Peak { + b := d.Bytes + r.Peak, r.PeakAt = &b, d.Day + } + } + if prev == nil { + return r, true + } + a, b := *prev, r + a.MeasuredAt, b.MeasuredAt = nil, nil + return r, !reflect.DeepEqual(a, b) +} From bb993dd606412166ddec503a35e61fee327cd9a4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:28:21 +0100 Subject: [PATCH 41/52] Keep the builder's history as its own log that readers declare and follow, the memberships as the grants part's state, and refuse a module's own stream, so every module's data is in the bus's predefined forms and readable by its tools (issue 1955, ADR 0571 clauses 1, 4, 5, 7) --- go/buscompose/buscompose_test.go | 33 +++++- go/buscompose/log.go | 59 +++++++++- go/buscompose/membership.go | 7 +- go/buscompose/nats.go | 20 ++++ go/buscompose/users.go | 10 +- go/catalogue/seats_declared.go | 2 + go/catalogue/streams.go | 28 +++++ go/catalogue/streams_test.go | 8 ++ go/delivery/control_records_test.go | 42 ++++++- go/delivery/history_records.go | 149 +++++++++++++++++++----- go/log.go | 106 ++++++++++++++++- go/log_test.go | 56 ++++++++- go/process/fixture_integration_test.go | 8 ++ go/process/log.go | 152 +++++++++++++++++++++++++ go/process/state_integration_test.go | 38 +++++++ go/stdio.go | 28 ++++- 16 files changed, 695 insertions(+), 51 deletions(-) create mode 100644 go/process/log.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 5e356b4..a83b771 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -59,6 +59,19 @@ func beyondTheControllers(user string) map[string]bool { // mayBeyondTheControllers is what the copy may grant beyond the controller's where the records allow it, and need not // where they do not: a node's ask of the vault's `own` while the vault serves it (issue 1126, ADR 0550 §5). func mayBeyondTheControllers(user string) map[string]bool { + // A runtime's read of its membership from the grants part's state (novox/hq ADR 0571 clause 7): its own key, or the + // tool runner's node's. + if parts := strings.Split(user, "."); len(parts) == 2 && parts[0] != "node" { + key := user + if parts[1] == RuntimeModule { + key = parts[0] + ".*" + } + out := map[string]bool{} + for _, s := range readGrants(GrantsMembershipsBucket, key) { + out[s] = true + } + return out + } if _, ok := strings.CutPrefix(user, "node."); !ok { return nil } @@ -618,6 +631,11 @@ func healthChecks(t *testing.T) { // ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them. {Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}}, {Module: "restic", Reads: []string{"disk-load.readings"}}, + // ADR 0571 clauses 1, 9: the indexer reads the builder's log through its own consumer; the catalogue module asks + // the builder's tool. + {Module: delivery.BuilderModule, Logs: []Log{{Module: delivery.BuilderModule, Name: delivery.BuilderLog, MaxAgeDays: 30}}}, + {Module: delivery.IndexerModule, Reads: []string{delivery.BuildLogRead}}, + {Module: "mesh-catalog", Invokes: []string{delivery.BuilderModule + "." + delivery.BuildsSinceTool}}, {Module: "telegram", Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, Processes: []ProcessPrincipal{{Process: "w", Queues: []QueueOwner{{Process: "w", Stream: "M_WORK"}, {Process: "w", Stream: "KV_x"}}}}}, {Module: "nats", SnapshotsTheBus: true, Processes: []ProcessPrincipal{{Process: "nats-tools", @@ -634,7 +652,14 @@ func healthChecks(t *testing.T) { "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, - "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data"}, + "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", + // ADR 0571 clause 7: its own membership from the grants part's state. + "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"}, + "a.mesh-delivery-indexer": {"$JS.API.CONSUMER.CREATE.LOG_mesh-delivery-builder_build-log.r1.mesh.log.mesh-delivery-builder.build-log.letta", + "$JS.API.CONSUMER.MSG.NEXT.LOG_mesh-delivery-builder_build-log.r1", "$JS.API.CONSUMER.DELETE.LOG_mesh-delivery-builder_build-log.r1", + "$JS.API.DIRECT.GET.LOG_mesh-delivery-builder_build-log.mesh.log.mesh-delivery-builder.build-log.letta"}, + "a.mesh-delivery-builder": {"mesh.log.mesh-delivery-builder.build-log.letta"}, + "a.mesh-catalog": {"mesh.mod.mesh-delivery-builder.tool.builder_builds_since"}, WatchmanModule: {"$JS.API.CONSUMER.CREATE.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.MSG.NEXT.DEAD_LETTER_NOTICES." + WatchmanModule, "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", "$JS.API.STREAM.MSG.GET.DEAD_LETTERS"}, WatchmanModule + " sources": {"$JS.API.STREAM.MSG.GET.EVENTS", "$JS.API.STREAM.INFO.SEAT_TELEGRAM_SENDER", @@ -678,6 +703,12 @@ func healthChecks(t *testing.T) { "a.nats": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, "a.shop.nats-tools": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-conditions_conditions.x"}, } + // ADR 0571 clause 1: a read naming another module's log is issued to the reader as a log it reads, never writes. + if m := MembershipFor("a", Declared{Module: delivery.IndexerModule, Reads: []string{delivery.BuildLogRead, "x.y"}}, + Placements{Logs: map[string]bool{delivery.BuildLogRead: true}}); len(m.Logs) != 1 || m.Logs[0].Writes || + m.Logs[0].Stream != "LOG_mesh-delivery-builder_build-log" || m.Logs[0].Name != delivery.BuildLogRead { + t.Errorf("the indexer's membership issues the logs %+v", m.Logs) + } // ADR 0565 clauses 6, 7 (issue 1953 L5-S4): a node's engine writes its own packages key and no other node's; only the // assigner writes its policies. for _, u := range users { diff --git a/go/buscompose/log.go b/go/buscompose/log.go index 9f1370f..d179b2b 100644 --- a/go/buscompose/log.go +++ b/go/buscompose/log.go @@ -15,8 +15,10 @@ import ( // A log follows a bucket in every respect (ADR 0201): the module names it locally, the mesh derives // its stream and subjects, the controller creates it from the catalogue on every raise and from // registration, never a module, and **no path of the mesh removes one** — a log whose declaration -// is gone is reported, as a bucket is. Unlike a bucket, a log is its owner's alone: no other module -// reads it, so there is no read of a log to grant or issue. +// is gone is reported, as a bucket is. Only its owner appends to it. Another module reads it when its manifest's +// `reads` names it, `.` (novox/hq ADR 0571 clause 1; the lead, 2026-10-29): by direct gets, and through an +// ordered consumer of its own it creates and lets go of to read from a time or the newest per key and then follow new +// entries; never a durable consumer, never a write. // // Pure, but for the stream's configuration, which is the server's own type so that what is tested // is exactly what is sent; jetstream.go is the part that asks a server. @@ -138,6 +140,59 @@ func logGrants(module string, names []string) []string { return out } +// DeclaredLogs is every log a module placed anywhere declares, as a read names it: `.`. +func DeclaredLogs(r Records) map[string]bool { + out := map[string]bool{} + for _, ds := range r.Assigned { + for _, d := range ds { + for _, l := range d.Logs { + out[d.Module+"."+l.Name] = true + } + } + } + return out +} + +// withReadLogs is d with ReadLogs set: of its reads, those naming a log of logs. +func withReadLogs(d Declared, logs map[string]bool) Declared { + d.ReadLogs = nil + for _, r := range d.Reads { + if logs[strings.TrimSpace(r)] { + d.ReadLogs = append(d.ReadLogs, strings.TrimSpace(r)) + } + } + return d +} + +// readLogGrants is what a reader of other modules' logs publishes, for each `.` read: the stream's info, +// direct gets (an entry by sequence, the last of a key), and an ordered consumer of its own (created with any name and +// filter, pulled, read and deleted). No append, no purge, no delete of an entry: the log's stream refuses those anyway. +func readLogGrants(reads []string) []string { + var out []string + for _, r := range reads { + module, name, ok := strings.Cut(r, ".") + if !ok || !safeSubject.MatchString(module) || !safeSubject.MatchString(name) { + continue + } + stream := LogStreamName(module, name) + out = append(out, "$JS.API.STREAM.INFO."+stream, "$JS.API.DIRECT.GET."+stream, "$JS.API.DIRECT.GET."+stream+".>", + "$JS.API.CONSUMER.CREATE."+stream+".>", "$JS.API.CONSUMER.MSG.NEXT."+stream+".*", + "$JS.API.CONSUMER.INFO."+stream+".*", "$JS.API.CONSUMER.DELETE."+stream+".*") + } + return out +} + +// readLogsIssuedFor is every other module's log a module's code reads, as its membership lists them: by the read's +// name, never written. +func readLogsIssuedFor(d Declared) []LogIssued { + var out []LogIssued + for _, r := range d.ReadLogs { + module, name, _ := strings.Cut(r, ".") + out = append(out, LogIssued{Name: r, Stream: LogStreamName(module, name), Subject: LogSubject(module, name)}) + } + return out +} + // logNames is the local names of a module's logs. func logNames(logs []Log) []string { out := make([]string, 0, len(logs)) diff --git a/go/buscompose/membership.go b/go/buscompose/membership.go index d28ff68..4b4b716 100644 --- a/go/buscompose/membership.go +++ b/go/buscompose/membership.go @@ -24,6 +24,9 @@ type Placements struct { Interchangeable map[string]bool // Kinds is every kind held of a kinded bench, by whom and with what capabilities (ADR 0259 §5). Kinds []KindHeld + // Logs is every log a module placed anywhere declares, `.` (DeclaredLogs): a read naming one is issued + // as a log read (novox/hq ADR 0571 clause 1). + Logs map[string]bool } // AnswersForTheModule says whether an instance of a module on one machine is issued the module's @@ -58,7 +61,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership { } } m.State = stateIssuedFor(d, node) - m.Logs = logsIssuedFor(d) + m.Logs = append(logsIssuedFor(d), readLogsIssuedFor(withReadLogs(d, where.Logs))...) m.Streams = streamsIssuedFor(d) m.Hears = HeardIssuedFor(node, d) m.Waits = waitsIssuedFor(d) @@ -103,7 +106,7 @@ func MembershipFor(node string, d Declared, where Placements) Membership { // PlacementsOf reads where everything runs from the records the bus's accounts are composed from. func PlacementsOf(r Records, interchangeable map[string]bool) Placements { - p := Placements{Nodes: map[string][]string{}, Interchangeable: interchangeable} + p := Placements{Nodes: map[string][]string{}, Interchangeable: interchangeable, Logs: DeclaredLogs(r)} for node, declared := range r.Assigned { for _, d := range declared { p.Nodes[d.Module] = append(p.Nodes[d.Module], node) diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 83378f1..61bdfa9 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -181,6 +181,8 @@ type Principal struct { KeyedReads []KeyedRead // Logs is the local names of the logs this principal's module keeps (novox/hq ADR 0297). Logs []string + // ReadLogs are, of Reads, those naming another module's log (novox/hq ADR 0571 clause 1). + ReadLogs []string // Streams are the streams this principal's module owns, and Hears the module streams it hears with the // subjects it was granted (novox/hq ADR 0363). Streams []ModuleStream @@ -1063,6 +1065,9 @@ func PermissionsFor(p Principal) (Permissions, error) { // directly from the stream and followed live. Nothing else's. sub = append(sub, MembershipSubject(p.Node, p.Module)) pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+"."+MembershipSubject(p.Node, p.Module)) + // And from the grants part's state memberships, its own key (novox/hq ADR 0571 clause 7): the read that replaces + // the stream's at the lease write, when the grants part stops publishing there. + pub = append(pub, readGrants(GrantsMembershipsBucket, MembershipKey(p.Node, p.Module))...) // 1b. The tools it calls, if its manifest says it calls any (novox/hq ADR 0152). The same // grant a person gets and derived the same way, so "what may this module ask" is @@ -1180,6 +1185,7 @@ func PermissionsFor(p Principal) (Permissions, error) { sub = append(sub, stateCreatedHeard(kept)...) // And its logs (novox/hq ADR 0297): appended to and read directly, its own alone. pub = append(pub, logGrants(p.Module, p.Logs)...) + pub = append(pub, readLogGrants(p.ReadLogs)...) // 6. Its traffic on seats that name their caller or their kind, ask proofs or keep records // (novox/hq ADR 0259 §3). @@ -1227,6 +1233,7 @@ func PermissionsFor(p Principal) (Permissions, error) { // that varies is the module, so the pattern is the machine's own assignments. sub = append(sub, "mesh.assignment."+p.Node+".*") pub = append(pub, "$JS.API.DIRECT.GET."+AssignmentsStream+".mesh.assignment."+p.Node+".*") + pub = append(pub, readGrants(GrantsMembershipsBucket, p.Node+".*")...) // And that it is there (novox/hq to-be 45 §3, S11): its own heartbeat, under its machine's // name and no other's, on core NATS like the host's. pub = append(pub, "mesh.control."+p.Node+".tools-alive") @@ -1283,6 +1290,7 @@ func PermissionsFor(p Principal) (Permissions, error) { // runtime's to keep, from the logs each membership lists. for _, d := range p.Carries { pub = append(pub, logGrants(d.Module, logNames(d.Logs))...) + pub = append(pub, readLogGrants(d.ReadLogs)...) } // **And the event families a session may wait on** (novox/hq ADR 0367 §2): the tool runner's module's own, // named one by one in its manifest, as a plain subscription — no consumer is made for them, and no @@ -1728,6 +1736,18 @@ func CreateConsumerGrants(consumers []ModuleConsumer) []string { // part, which composes them from the manifests and placement. It defines no stream and is granted no other subject here. const GrantsModule = "mesh-grants" +// The grants part's state memberships (novox/hq ADR 0571 clause 7): one key per assignment, `.`, its +// membership (Membership) as the assignments stream carries it, written only when it differs. A module's runtime reads +// its own key, the tool runner its node's; the grants part alone writes it. +const ( + GrantsMembershipsState = "memberships" + GrantsMembershipsBucket = GrantsModule + "_" + GrantsMembershipsState + GrantsMembershipsRead = GrantsModule + "." + GrantsMembershipsState +) + +// MembershipKey is an assignment's key in the grants part's memberships: `.`. +func MembershipKey(node, module string) string { return node + "." + module } + // The bus's own events, imported into MESH from the default system account `$SYS` under subjects of MESH's own // (novox/hq ADR 0431 §1, issue 1147): the server says an account's connections opening and closing only to the system // account, and the list of an account's connections only to a request made there. `$SYS` holds no user, so nobody diff --git a/go/buscompose/users.go b/go/buscompose/users.go index ca23cf5..c370f5d 100644 --- a/go/buscompose/users.go +++ b/go/buscompose/users.go @@ -64,6 +64,9 @@ type Declared struct { // ReadsRefused are the reads its manifest names that the owner's announcement does not make readable (novox/hq ADR // 0495 clause 3, issue 1790): granted nothing, kept to be said. ReadsRefused []string + // ReadLogs are, of its reads, those naming another module's log (`.`, novox/hq ADR 0571 clause 1): filled + // by Users and PlacementsOf from the logs the modules placed anywhere declare, never by a caller. + ReadLogs []string // KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is // granted for the pieces the modules beside it offer (novox/hq ADR 0260). KeyedReads []KeyedRead @@ -157,6 +160,7 @@ func Users(r Records) ([]Principal, error) { AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig), FilesFor: sortedCopy(r.Nodes)}} served := servedReadsOf(r) + logs := DeclaredLogs(r) grants := -1 for _, node := range sortedCopy(r.Nodes) { witness := false @@ -198,7 +202,7 @@ func Users(r Records) ([]Principal, error) { for _, d := range r.Assigned[node] { // A state with named readers is read by them alone (ADR 0547 clause 9): a read named for another module is // granted nothing, here and in the runtime carrying the module. - d = withNamedReaders(d) + d = withReadLogs(withNamedReaders(d), logs) // **A process account is its own, delivered in its own secret** (broker-, novox/hq // ADR 0436 §1, §2), never the module's `broker` — so it is composed whether or not the module // itself has an account, and is never carried by the machine's runtime, which runs as the @@ -226,7 +230,7 @@ func Users(r Records) ([]Principal, error) { Kind: KindModule, Node: node, Module: d.Module, Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves, Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, Invokes: d.Invokes, - State: stateNames(d.State), Reads: d.Reads, SnapshotsTheBus: d.SnapshotsTheBus, + State: stateNames(d.State), Reads: d.Reads, ReadLogs: d.ReadLogs, SnapshotsTheBus: d.SnapshotsTheBus, PerMachine: perMachineNames(d.State), KeyedReads: d.KeyedReads, Logs: logNames(d.Logs), Streams: d.Streams, Hears: ComposedHears(d), Queues: d.Queues, }) @@ -250,7 +254,7 @@ func Users(r Records) ([]Principal, error) { var carried []Declared for _, d := range r.Assigned[node] { if d.RunsAs == "" && runnerServes(d) { - d = withNamedReaders(d) + d = withReadLogs(withNamedReaders(d), logs) // Never a restricted stream for what the tool runner carries (novox/hq ADR 0376 §1). d.Hears = ComposedHears(d) carried = append(carried, d) diff --git a/go/catalogue/seats_declared.go b/go/catalogue/seats_declared.go index 90087e2..b8342e7 100644 --- a/go/catalogue/seats_declared.go +++ b/go/catalogue/seats_declared.go @@ -285,6 +285,8 @@ func CatalogueProblems(shelf Shelf) []string { // Every offer says its credential's form, never left to a default (the lead, 2026-10-29, for novox/hq ADR 0570). for _, module := range shelfOrder(shelf) { problems = append(problems, CredentialFormProblems(shelf[module])...) + // And its data in the bus's predefined forms, a stream of its own refused (the lead, 2026-10-29). + problems = append(problems, PredefinedFormProblems(shelf[module])...) } // Who declares what, and who declared it first. diff --git a/go/catalogue/streams.go b/go/catalogue/streams.go index 15c6c2b..fa166e2 100644 --- a/go/catalogue/streams.go +++ b/go/catalogue/streams.go @@ -630,3 +630,31 @@ func uncovered(patterns []string) []string { slices.Sort(out) return out } + +// **A module keeps its data in the bus's predefined forms** (the lead and the operator, 2026-10-29; the operator's rule of +// 2026-10-22, "data storage is done using our predefined nats bus options"): a state, a log or an object store, which the +// runtime reads and writes for its tools and other modules read by a declared `reads`. A stream of its own is outside +// them and refused, naming it. StreamsKeptUntilMoved are the streams declared before this check, each allowed by name +// until its module moves it; the list only shrinks. +var StreamsKeptUntilMoved = map[string][]string{ + "brother": {"device-asks"}, + "mesh-build-go": {"toolchains-go"}, + "mesh-build-node": {"toolchains-node"}, + "mesh-mail": {"mail", "mail-sends"}, + "only-office": {"convert-asks"}, + "mesh-delivery-assigner": {"placement"}, + "mesh-delivery-checker": {"checks"}, + "mesh-delivery-forge-bridge": {"forge"}, +} + +// PredefinedFormProblems refuses each stream a module declares that StreamsKeptUntilMoved does not name for it. +func PredefinedFormProblems(m Manifest) []string { + var out []string + for _, s := range m.Streams { + if !slices.Contains(StreamsKeptUntilMoved[m.Module], s.Name) { + out = append(out, fmt.Sprintf("%s keeps the stream %q, outside the bus's predefined forms: keep it as a state, "+ + "a log or an object store, which the runtime reads for the module's tools and others read by `reads`", m.Module, s.Name)) + } + } + return out +} diff --git a/go/catalogue/streams_test.go b/go/catalogue/streams_test.go index a5971e3..cb02362 100644 --- a/go/catalogue/streams_test.go +++ b/go/catalogue/streams_test.go @@ -96,6 +96,14 @@ func TestTwoModulesCannotDeclareOneStream(t *testing.T) { if got := StreamNameProblems(Shelf{"a": a}); len(got) != 0 { t.Fatalf("one owner refused: %v", got) } + // And a stream of its own is outside the bus's predefined forms (the lead, 2026-10-29): refused, naming it, unless it + // is one kept by name until its module moves it. + if !strings.Contains(got, `a keeps the stream "mail", outside the bus's predefined forms`) { + t.Fatalf("a stream of its own was not refused: %s", got) + } + if got := PredefinedFormProblems(Manifest{Module: "mesh-mail", Streams: a.Streams}); len(got) != 0 { + t.Fatalf("a stream kept until it moves was refused: %v", got) + } } // The setting's value names who hears each key: a module wherever it runs, or on one machine (ADR 0363 §4). diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index 94683f4..2c64037 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -58,9 +58,13 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { } // ADR 0571 clause 4: who holds which seat, from placement; a node seat by its node. held := SeatHoldersOf(map[string]Placement{ - "ace": {Node: "ace", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}}, - "novox": {Node: "novox", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"}}}, + "ace": {Node: "ace", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "node-files", Scope: "node", Module: "mesh-files"}}}, + "novox": {Node: "novox", Holds: []Holding{{Claim: "mesh-dns-resolver", Scope: "mesh", Module: "dnsmasq"}, {Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, + {Claim: "photo-gallery", Scope: "mesh", Module: "photos"}}}, }) + if o := held["photo-gallery"]; !o.Outside || len(o.Holders) != 1 || held["mesh-broker"].Outside { + t.Errorf("a claim outside the set is %+v", o) + } keys := map[string][]byte{} for k, v := range held { keys[k], _ = json.Marshal(v) @@ -68,9 +72,19 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { if h, err := HoldersOf(keys, "mesh-dns-resolver", ""); err != nil || len(h) != 2 || h[0].Node != "ace" { t.Errorf("a replicated seat's holders are %v, %v", h, err) } - if h, _ := HoldersOf(keys, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 3 { + if h, _ := HoldersOf(keys, "node-files", "ace"); len(h) != 1 || h[0].Module != "mesh-files" || len(held) != 4 { t.Errorf("a node seat's holder is %v, keys %v", h, held) } + // ADR 0571 clauses 4, 5: the modules and the machines, from the indexer's and the placement seat's keys. + if m, err := ModuleNames(map[string][]byte{"letta": nil, "gitea": nil}); err != nil || strings.Join(m, ",") != "gitea,letta" { + t.Errorf("the modules are %v, %v", m, err) + } + if _, err := ModuleNames(map[string][]byte{"a.b": nil}); err == nil { + t.Error("a key that is no module's was read as one") + } + if n := MachineNames(map[string][]byte{"placement.novox": nil, "placement.ace": nil, "proposal.p1": nil}); strings.Join(n, ",") != "ace,novox" { + t.Errorf("the machines are %v", n) + } if h, _ := HoldersOf(keys, "node-files", "novox"); h != nil { t.Errorf("a node seat nobody holds there is held by %v", h) } @@ -79,7 +93,8 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { start := BuildLogEntry{Event: BuildStartedEvent, Module: "letta", FP: "src2:a", Source: "abc", Repository: "novox/mesh-catalog", Directory: "modules/letta", Commit: "abc123", Node: "ace", At: at} end := start - end.Event, end.At, end.Outcome, end.Artifacts = BuildEndedEvent, at.Add(3*time.Minute), "done", []BuiltDigest{{Name: "image", Digest: "sha256:1"}} + end.Event, end.At, end.Outcome, end.Manifest, end.Against = BuildEndedEvent, at.Add(3*time.Minute), "done", "sha256:m", []string{"artifact-store://base/image@sha256:b"} + end.Made = []BuiltArtifact{{Name: "image", Kind: "image", Digest: "sha256:1", Reference: "artifact-store://letta/image@sha256:1"}} for _, e := range []BuildLogEntry{start, end} { raw, err := EncodeBuildLogEntry(e) back, derr := DecodeBuildLogEntry(raw) @@ -91,7 +106,24 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { if d, ok := BuildDuration(start, end); !ok || d != 3*time.Minute { t.Errorf("the build took %s, %v", d, ok) } - for _, edit := range []func(*BuildLogEntry){func(e *BuildLogEntry) { e.Outcome = "" }, func(e *BuildLogEntry) { e.Commit = "" }} { + older := end + older.At, older.FP = at.Add(-time.Hour), "src2:b" + other := end + other.Module = "mesh-catalog" + if n := NewestPerModule([]BuildLogEntry{older, end, other}); n["letta"].FP != "src2:a" || len(n) != 2 { + t.Errorf("each module's newest is %v", n) + } + if e := EndedSince([]BuildLogEntry{end, start, older}, at.Add(-2*time.Hour)); len(e) != 2 || e[0].FP != "src2:b" { + t.Errorf("the ended builds since are %v", e) + } + if k, err := BuildLogKey("letta"); k != "letta" || err != nil || BuildLogRead != "mesh-delivery-builder.build-log" { + t.Errorf("the key is %q, %v", k, err) + } + if _, err := BuildLogKey("a.b"); err == nil { + t.Error("a module with a dot was taken as a key") + } + for _, edit := range []func(*BuildLogEntry){func(e *BuildLogEntry) { e.Outcome = "" }, func(e *BuildLogEntry) { e.Commit = "" }, + func(e *BuildLogEntry) { e.Event, e.Outcome, e.Against = BuildStartedEvent, "", nil }} { bad := end edit(&bad) if _, err := EncodeBuildLogEntry(bad); err == nil { diff --git a/go/delivery/history_records.go b/go/delivery/history_records.go index 6284549..714ff85 100644 --- a/go/delivery/history_records.go +++ b/go/delivery/history_records.go @@ -5,6 +5,8 @@ import ( "fmt" "sort" "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // **Who holds which seat, and the builder's log** (novox/hq ADR 0571 clauses 1 and 4, issue 1955). @@ -24,12 +26,15 @@ type SeatHolder struct { } // SeatHolders is one key of seat-holders: the seat, its scope, the node for a node seat's key, and its holders -// (several only for a replicated mesh seat), sorted. +// (several only for a replicated mesh seat), sorted. Outside says the claim names no seat of the set (catalogue.Seats): +// a module registered before the set closed, or built outside the indexed repositories, keeps its claim listed, as the +// controller's `seats` listed it under outside. type SeatHolders struct { V int `json:"v"` Seat string `json:"seat"` Scope string `json:"scope"` Node string `json:"node,omitempty"` + Outside bool `json:"outside,omitempty"` Holders []SeatHolder `json:"holders"` } @@ -49,7 +54,8 @@ func SeatHoldersKey(seat, node string) (string, error) { } // SeatHoldersOf is the state composed from every node's placement (ADR 0567 §1): each holding by the seat it claims, -// keyed as SeatHoldersKey says, with the module holding it and its node. Deterministic, so a writer writes only a key +// keyed as SeatHoldersKey says, with the module holding it and its node. A claim naming a seat's former name is keyed by +// the seat's name (ADR 0122); one naming no seat is kept, marked Outside. Deterministic, so a writer writes only a key // that differs. func SeatHoldersOf(placements map[string]Placement) map[string]SeatHolders { out := map[string]SeatHolders{} @@ -64,12 +70,17 @@ func SeatHoldersOf(placements map[string]Placement) map[string]SeatHolders { if h.Scope == "node" { node = n } - key, err := SeatHoldersKey(h.Claim, node) + seat, known := catalogue.SeatNamed(h.Claim) + name := h.Claim + if known { + name = seat.Name + } + key, err := SeatHoldersKey(name, node) if err != nil { continue } sh := out[key] - sh.V, sh.Seat, sh.Scope, sh.Node = Version, h.Claim, h.Scope, node + sh.V, sh.Seat, sh.Scope, sh.Node, sh.Outside = Version, name, h.Scope, node, !known sh.Holders = append(sh.Holders, SeatHolder{Module: h.Module, Node: n}) out[key] = sh } @@ -110,39 +121,89 @@ func HoldersOf(keys map[string][]byte, seat, node string) ([]SeatHolder, error) return nil, nil } -// The builder's log (clause 1): a stream of its own, `log`, one entry when a build of a fingerprint starts and one when -// it ends, kept 30 days. +// The builder's log (clause 1; the lead and the operator, 2026-10-29): the builder's own module log (ADR 0297) `build-log`, +// kept 30 days (max-age-days), one entry under the key `` when a build of a fingerprint starts and one when it +// ends, so the newest entry of a key is the module's newest start or end. The builder's tools read it through the +// runtime's log verbs; another module reads it by naming BuildLogRead in its manifest's `reads`: a bundle through the +// runtime (stdio.Log(BuildLogRead)), a process on its own credential (process.Conn.ReadLog), each by newest per key, +// from a time, or following new entries. The catalogue module asks the builder's tool BuildsSinceTool instead. const ( - BuilderLogStream = "log" + BuilderLog = "build-log" + BuildLogRead = BuilderModule + "." + BuilderLog BuildStartedEvent = "build.started" BuildEndedEvent = "build.ended" // BuilderLogDays is how long the log keeps its entries. BuilderLogDays = 30 + // BuildsSinceTool is the builder's tool answering every ended build since a time, in order (clause 9): its + // argument `since` (RFC 3339), its answer BuildsSince. + BuildsSinceTool = "builder_builds_since" ) -// BuiltDigest is one artifact a build ended with, by its name and digest. -type BuiltDigest struct { - Name string `json:"name"` - Digest string `json:"digest"` +// BuildLogKey is the key of a module's entries in the builder's log: the module's name, refused when a log key could not +// hold it. +func BuildLogKey(module string) (string, error) { + if err := checkName("module", module); err != nil { + return "", err + } + return module, nil } // BuildLogEntry is one entry of the builder's log: the module, the fingerprint, the source revision, where the module is -// built from (its repository, its directory there and the commit), the build node and the time; at the end, the outcome -// and each artifact's digest. The repository, directory and commit are what the indexer's PathOf, the outside-built -// registrations and the assigner's running commits read (the lead, 2026-10-29). +// built from (its repository, its directory there, the commit and the ref), the build node and the time; at the end, +// the outcome, the manifest built (its digest, read from the indexer's state `manifests` under ManifestsKey), what it was +// built against (the bases' references, as the controller's built-before said them) and each artifact made +// (BuiltArtifact, the done key's: its name, kind, digest and reference). The repository, directory and commit are what +// the indexer's PathOf, the outside-built registrations and the assigner's running commits read; the manifest, against +// and made what the catalogue module registers a build from (the lead, 2026-10-29). type BuildLogEntry struct { - V int `json:"v"` - Event string `json:"event"` - Module string `json:"module"` - FP string `json:"fp"` - Source string `json:"source"` - Repository string `json:"repository"` - Directory string `json:"directory"` - Commit string `json:"commit"` - Node string `json:"node"` - At time.Time `json:"at"` - Outcome string `json:"outcome,omitempty"` - Artifacts []BuiltDigest `json:"artifacts,omitempty"` + V int `json:"v"` + Event string `json:"event"` + Module string `json:"module"` + FP string `json:"fp"` + Source string `json:"source"` + Repository string `json:"repository"` + Directory string `json:"directory"` + Commit string `json:"commit"` + Ref string `json:"ref,omitempty"` + Node string `json:"node"` + At time.Time `json:"at"` + Outcome string `json:"outcome,omitempty"` + Manifest string `json:"manifest,omitempty"` + Against []string `json:"against,omitempty"` + Made []BuiltArtifact `json:"made,omitempty"` +} + +// BuildsSince is the answer of BuildsSinceTool: every ended build since a time, oldest first, and whether the log held +// more than one answer carries (then ask again from the last one's time). +type BuildsSince struct { + Since time.Time `json:"since"` + Builds []BuildLogEntry `json:"builds"` + More bool `json:"more,omitempty"` +} + +// NewestPerModule is each module's newest entry of entries (one kind, read from the last per subject or whole), by +// module: the reader's helper for the five newest and the running commits. +func NewestPerModule(entries []BuildLogEntry) map[string]BuildLogEntry { + out := map[string]BuildLogEntry{} + for _, e := range entries { + if have, ok := out[e.Module]; !ok || e.At.After(have.At) { + out[e.Module] = e + } + } + return out +} + +// EndedSince is, of entries, the ended builds at or after since, oldest first: the reader's helper for history and for +// BuildsSinceTool's answer. +func EndedSince(entries []BuildLogEntry, since time.Time) []BuildLogEntry { + var out []BuildLogEntry + for _, e := range entries { + if e.Event == BuildEndedEvent && !e.At.Before(since) { + out = append(out, e) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out } // EncodeBuildLogEntry is e as JSON at Version; it refuses an entry a reader could not place. @@ -155,8 +216,8 @@ func EncodeBuildLogEntry(e BuildLogEntry) ([]byte, error) { return nil, fmt.Errorf("delivery: a build log entry names its module, fingerprint, repository, commit, node and time") case e.Event == BuildEndedEvent && e.Outcome == "": return nil, fmt.Errorf("delivery: a build's end names its outcome") - case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Artifacts) > 0): - return nil, fmt.Errorf("delivery: a build's start names no outcome and no artifact") + case e.Event == BuildStartedEvent && (e.Outcome != "" || len(e.Made) > 0 || e.Manifest != "" || len(e.Against) > 0): + return nil, fmt.Errorf("delivery: a build's start names no outcome, manifest, base or artifact") } return json.Marshal(e) } @@ -182,3 +243,35 @@ func BuildDuration(start, end BuildLogEntry) (time.Duration, bool) { } return end.At.Sub(start.At), true } + +// **Which modules exist and which machines the mesh has, from the bus** (novox/hq ADR 0571 clauses 4, 5; the lead for a6, +// 2026-10-29): the tool runner reads the modules from the indexer's state `modules` and the machines from the placement +// seat's keys, and no longer asks the controller's `modules` or `nodes`. + +// ModuleNames is every module the indexer holds, sorted, from its state `modules` as a reader holds its keys (key to +// value; a deleted key absent). A key that is not a module's is refused, naming it. +func ModuleNames(keys map[string][]byte) ([]string, error) { + out := make([]string, 0, len(keys)) + for k := range keys { + module, err := ParseModulesKey(k) + if err != nil { + return nil, err + } + out = append(out, module) + } + sort.Strings(out) + return out, nil +} + +// MachineNames is every machine the placement seat places modules on, sorted, from its state `placement` as a reader +// holds its keys: each `placement.`. Keys of the state's other kinds (a proposal's) are passed over. +func MachineNames(keys map[string][]byte) []string { + out := []string{} + for k := range keys { + if node, err := ParsePlacementKey(k); err == nil { + out = append(out, node) + } + } + sort.Strings(out) + return out +} diff --git a/go/log.go b/go/log.go index 7b1030d..aad2169 100644 --- a/go/log.go +++ b/go/log.go @@ -9,8 +9,9 @@ import ( // Log — a module's record of operations on the bus, kept as long as the log is (novox/hq ADR 0297). Not the // current value: that is state, and a log's state is its fold. Each entry is one operation on one key, its // sequence in the log is its identity, and the order of the log is the order of the operations. A module -// declares each log it owns in its manifest (`logs`) and names it here as it named it there; a log is its -// owner's alone, and the runtime refuses, with the reason, one it did not issue. +// declares each log it owns in its manifest (`logs`) and names it here as it named it there; only its owner appends. +// Another module's log is read by naming it `.` in the manifest's `reads` and here (novox/hq ADR 0571 +// clause 1): read, newest per key, from a time, and followed. The runtime refuses, with the reason, one it did not issue. // LogEntry is one operation on one key, as the log holds it. type LogEntry struct { @@ -40,7 +41,7 @@ type LogInfo struct { // KeptLog is one log this module owns. type KeptLog struct{ name string } -// Log is a log this module owns, by its local name. +// Log is a log this module owns, by its local name, or reads, `.`. func Log(name string) KeptLog { return KeptLog{name: name} } const logTimeout = 30 * time.Second @@ -121,3 +122,102 @@ func (l KeptLog) Info() (*LogInfo, error) { var i LogInfo return &i, json.Unmarshal(raw, &i) } + +// Newest is the newest entry of every key keys names (a key, `*` for one name, `**` for the rest; empty is every key), +// in the log's order: runtime verb `mesh/log.newest` {log, keys}, answered as a list of entries. +func (l KeptLog) Newest(keys string) ([]LogEntry, error) { + params := map[string]any{} + if keys != "" { + params["keys"] = keys + } + raw, err := l.ask("newest", params) + if err != nil { + return nil, err + } + var out []LogEntry + return out, json.Unmarshal(raw, &out) +} + +// Since is one page of the entries from a time on, as Read pages them: runtime verb `mesh/log.read` with `since` (RFC +// 3339) in place of `from`. +func (l KeptLog) Since(at time.Time, keys string, limit int) (*LogPage, error) { + if limit < 0 { + return nil, fmt.Errorf("a read of %s asks for at least no entries: limit %d", l.name, limit) + } + params := map[string]any{"since": at.UTC().Format(time.RFC3339Nano)} + if keys != "" { + params["keys"] = keys + } + if limit > 0 { + params["limit"] = limit + } + raw, err := l.ask("read", params) + if err != nil { + return nil, err + } + var p LogPage + return &p, json.Unmarshal(raw, &p) +} + +// LogFrom is where a log's watch starts: LogFromNewest (the newest entry of each key, then every new one), LogFromAll +// (every entry the log keeps), LogFromSince (every entry from a time) or LogFromNow (only new entries). +type LogFrom struct { + Kind string + Since time.Time +} + +// The kinds of LogFrom. +const ( + LogFromNewestKind = "newest" + LogFromAllKind = "all" + LogFromSinceKind = "since" + LogFromNowKind = "now" +) + +var ( + LogFromNewest = LogFrom{Kind: LogFromNewestKind} + LogFromAll = LogFrom{Kind: LogFromAllKind} + LogFromNow = LogFrom{Kind: LogFromNowKind} +) + +// LogFromSince starts a watch at a time. +func LogFromSince(at time.Time) LogFrom { return LogFrom{Kind: LogFromSinceKind, Since: at} } + +func (f LogFrom) params() (map[string]any, error) { + p := map[string]any{"from": f.Kind} + switch f.Kind { + case LogFromNewestKind, LogFromAllKind, LogFromNowKind: + case LogFromSinceKind: + p["since"] = f.Since.UTC().Format(time.RFC3339Nano) + default: + return nil, fmt.Errorf("a log's watch starts from newest, all, since or now, not %q", f.Kind) + } + return p, nil +} + +// Watch hands handle each entry of keys from where from says, in the log's order, and then every new one as it is +// appended, and returns once the entries there at its start have been handled: runtime verb `mesh/log.watch` {log, +// keys, from, since}, each entry then sent as `mesh/log` {watch, entry}. Event-driven: nothing reads the log again on +// a timer. The watch lasts as long as the process. +func (l KeptLog) Watch(keys string, from LogFrom, handle func(LogEntry) error) error { + params, err := from.params() + if err != nil { + return err + } + c, err := live() + if err != nil { + return err + } + id := c.newID() + c.mu.Lock() + c.logWatches[id] = handle + c.mu.Unlock() + params["log"], params["keys"] = l.name, keys + if _, err := c.askAs(id, "mesh/log.watch", params, logTimeout); err != nil { + c.mu.Lock() + delete(c.logWatches, id) + c.mu.Unlock() + return err + } + return nil +} diff --git a/go/log_test.go b/go/log_test.go index b650925..98bba28 100644 --- a/go/log_test.go +++ b/go/log_test.go @@ -58,12 +58,13 @@ func (r *logRuntime) answer(m message) map[string]any { From uint64 `json:"from"` Keys string `json:"keys"` Limit int `json:"limit"` + Since time.Time `json:"since"` } _ = json.Unmarshal(m.Params, &p) r.mu.Lock() defer r.mu.Unlock() r.asked = append(r.asked, m) - if p.Log != "changes" { + if p.Log != "changes" && p.Log != "builder.build-log" { return map[string]any{"error": map[string]any{"code": -32000, "message": "issues was issued no log " + p.Log}} } switch strings.TrimPrefix(m.Method, "mesh/log.") { @@ -83,7 +84,7 @@ func (r *logRuntime) answer(m message) map[string]any { page := map[string]any{"entries": []LogEntry{}, "next": 0, "last": len(r.entries)} var got []LogEntry for _, e := range r.entries { - if e.Sequence < p.From || !matches(p.Keys, e.Key) { + if e.Sequence < p.From || !matches(p.Keys, e.Key) || e.At.Before(p.Since) { continue } if len(got) == limit { @@ -96,6 +97,16 @@ func (r *logRuntime) answer(m message) map[string]any { page["entries"] = got } return map[string]any{"result": page} + case "newest": + var out []LogEntry + for _, e := range r.entries { + if matches(p.Keys, e.Key) && r.last(e.Key) == e.Sequence { + out = append(out, e) + } + } + return map[string]any{"result": out} + case "watch": + return map[string]any{"result": map[string]any{}} case "last": seq := r.last(p.Key) if seq == 0 { @@ -148,6 +159,13 @@ func logging(t *testing.T, r *logRuntime, run func() (any, error)) map[string]an a := r.answer(m) a["id"] = m.ID send(a) + // A watch's entries follow its answer, each named by the watch's id (ADR 0571 clause 1). + if m.Method == "mesh/log.watch" { + r.mu.Lock() + e := r.entries[len(r.entries)-1] + r.mu.Unlock() + send(map[string]any{"id": "entry-1", "method": "mesh/log", "params": map[string]any{"watch": m.ID, "entry": e}}) + } }(m) continue } @@ -193,7 +211,9 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { pages []*LogPage last, none *LogEntry info *LogInfo - narrowed *LogPage + narrowed, since *LogPage + newest []LogEntry + watched *LogEntry ) got := logging(t, r, func() (any, error) { changes := Log("changes") @@ -232,8 +252,27 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { if none, err = changes.Last("999"); err != nil { return nil, err } - info, err = changes.Info() - return "done", err + if info, err = changes.Info(); err != nil { + return nil, err + } + // ADR 0571 clause 1: another module's log, newest per key, from a time, and followed. + if newest, err = changes.Newest(""); err != nil { + return nil, err + } + if since, err = changes.Since(time.Date(2026, 10, 11, 18, 0, 5, 0, time.UTC), "", 0); err != nil { + return nil, err + } + heard := make(chan LogEntry, 1) + if err = Log("builder.build-log").Watch("letta", LogFromSince(time.Date(2026, 10, 11, 18, 0, 0, 0, time.UTC)), + func(e LogEntry) error { heard <- e; return nil }); err != nil { + return nil, err + } + select { + case e := <-heard: + watched = &e + case <-time.After(5 * time.Second): + } + return "done", nil }) if out, isError := text(t, got); isError || out != `"done"` { t.Fatalf("the tool answered %s", out) @@ -296,6 +335,13 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { if none != nil { t.Fatalf("a key with no entry has a last: %+v", none) } + if len(newest) != 5 || newest[0].Key != "402" || newest[1].Sequence != 3 || len(since.Entries) != 2 || since.Entries[0].Sequence != 5 { + t.Fatalf("the newest per key are %+v, since a time %+v", newest, since) + } + if method, p = r.params(t, 15); method != "mesh/log.watch" || p["log"] != "builder.build-log" || p["keys"] != "letta" || + p["from"] != "since" || p["since"] != "2026-10-11T18:00:00Z" || watched == nil || watched.Sequence != 6 { + t.Fatalf("a watch asked %s %v and heard %+v", method, p, watched) + } if info == nil || info.Entries != 6 || info.MaxBytes != 1<<30 || info.First != 1 || info.Last != 6 || info.Bytes == 0 { t.Fatalf("the log's info is %+v", info) } diff --git a/go/process/fixture_integration_test.go b/go/process/fixture_integration_test.go index ce92cfa..91ed0a3 100644 --- a/go/process/fixture_integration_test.go +++ b/go/process/fixture_integration_test.go @@ -43,6 +43,14 @@ var sendGrants = []string{ "mesh.seat.operator-channel.call.tell.novox~mesh-mail~send", "mesh.seat.operator-channel.tool.tell", "mesh.seat.operator-channel.tool.tell.*", + // And the builder's log, as buscompose's readLogGrants grants a declared read of it (novox/hq ADR 0571 clause 1). + "$JS.API.STREAM.INFO.LOG_mesh-delivery-builder_build-log", + "$JS.API.DIRECT.GET.LOG_mesh-delivery-builder_build-log", + "$JS.API.DIRECT.GET.LOG_mesh-delivery-builder_build-log.>", + "$JS.API.CONSUMER.CREATE.LOG_mesh-delivery-builder_build-log.>", + "$JS.API.CONSUMER.MSG.NEXT.LOG_mesh-delivery-builder_build-log.*", + "$JS.API.CONSUMER.INFO.LOG_mesh-delivery-builder_build-log.*", + "$JS.API.CONSUMER.DELETE.LOG_mesh-delivery-builder_build-log.*", } // grantedSendProcess is a test bus with JetStream, with two users: novox.mesh-mail.send, permissioned to diff --git a/go/process/log.go b/go/process/log.go new file mode 100644 index 0000000..176c408 --- /dev/null +++ b/go/process/log.go @@ -0,0 +1,152 @@ +package process + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// **Another module's log, read on this process's own credential** (novox/hq ADR 0571 clause 1; the lead, 2026-10-29): a +// log the module's manifest names in `reads`, `.`, read through an ordered consumer this connection creates +// and lets go of, never a durable one. Newest is the newest entry of each key, Since every entry from a time, and Watch +// those or every entry, then each new one as it is appended: event-driven, nothing reads the log again on a timer. The +// grants are buscompose's readLogGrants, given for a declared read of a log alone. + +// LogReader is one module's log this connection reads. +type LogReader struct { + Module, Name string + js func() (jetstream.JetStream, error) + closed func() bool +} + +// ReadLog is module's log name, read on this connection. +func (c *Conn) ReadLog(module, name string) LogReader { + return LogReader{Module: module, Name: name, closed: func() bool { return c.nc.IsClosed() }, + js: func() (jetstream.JetStream, error) { return jetstream.New(c.nc) }} +} + +func (l LogReader) stream() string { return "LOG_" + l.Module + "_" + l.Name } +func (l LogReader) subject() string { return "mesh.log." + l.Module + "." + l.Name } + +// filter is keys as a subject filter: a key, `*` for one name, `**` for the rest; empty is every key. +func (l LogReader) filter(keys string) string { + if keys == "" { + return l.subject() + ".>" + } + return l.subject() + "." + watchPattern(keys) +} + +func (l LogReader) config(keys string, from stdio.LogFrom) (jetstream.OrderedConsumerConfig, error) { + cfg := jetstream.OrderedConsumerConfig{FilterSubjects: []string{l.filter(keys)}} + switch from.Kind { + case stdio.LogFromNewestKind: + cfg.DeliverPolicy = jetstream.DeliverLastPerSubjectPolicy + case stdio.LogFromAllKind: + cfg.DeliverPolicy = jetstream.DeliverAllPolicy + case stdio.LogFromSinceKind: + at := from.Since + cfg.DeliverPolicy, cfg.OptStartTime = jetstream.DeliverByStartTimePolicy, &at + case stdio.LogFromNowKind: + cfg.DeliverPolicy = jetstream.DeliverNewPolicy + default: + return cfg, fmt.Errorf("%s's log %s: a read starts from newest, all, since or now, not %q", l.Module, l.Name, from.Kind) + } + return cfg, nil +} + +func (l LogReader) entry(m jetstream.Msg) (stdio.LogEntry, error) { + meta, err := m.Metadata() + if err != nil { + return stdio.LogEntry{}, err + } + return stdio.LogEntry{Sequence: meta.Sequence.Stream, Key: strings.TrimPrefix(m.Subject(), l.subject()+"."), + Value: append([]byte(nil), m.Data()...), At: meta.Timestamp}, nil +} + +// Newest is the newest entry of each key keys names, in the log's order. +func (l LogReader) Newest(keys string) ([]stdio.LogEntry, error) { + return l.collect(keys, stdio.LogFromNewest) +} + +// Since is every entry of keys from at on, in the log's order. +func (l LogReader) Since(at time.Time, keys string) ([]stdio.LogEntry, error) { + return l.collect(keys, stdio.LogFromSince(at)) +} + +func (l LogReader) collect(keys string, from stdio.LogFrom) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + stop, err := l.watch(keys, from, func(e stdio.LogEntry) error { out = append(out, e); return nil }, false) + if stop != nil { + stop() + } + return out, err +} + +// Watch hands handle each entry from where from says and then every new one, in the log's order, and returns once the +// entries there at its start have been handled. The watch lasts as long as the connection; an error from handle ends +// it, as the connection closing does. Before the entries at its start are handled, either is returned as the error. +func (l LogReader) Watch(keys string, from stdio.LogFrom, handle func(stdio.LogEntry) error) error { + _, err := l.watch(keys, from, handle, true) + return err +} + +func (l LogReader) watch(keys string, from stdio.LogFrom, handle func(stdio.LogEntry) error, follow bool) (func(), error) { + cfg, err := l.config(keys, from) + if err != nil { + return nil, err + } + js, err := l.js() + if err != nil { + return nil, err + } + bind, stopBind := context.WithTimeout(context.Background(), stateTimeout) + defer stopBind() + cons, err := js.OrderedConsumer(bind, l.stream(), cfg) + if err != nil { + return nil, fmt.Errorf("%s's log %s (stream %s): %w", l.Module, l.Name, l.stream(), err) + } + info, err := cons.Info(bind) + if err != nil { + return nil, fmt.Errorf("%s's log %s (stream %s): %w", l.Module, l.Name, l.stream(), err) + } + pending := info.NumPending + it, err := cons.Messages() + if err != nil { + return nil, fmt.Errorf("%s's log %s (stream %s): %w", l.Module, l.Name, l.stream(), err) + } + for ; pending > 0; pending-- { + m, err := it.Next() + if err == nil { + var e stdio.LogEntry + if e, err = l.entry(m); err == nil { + err = handle(e) + } + } + if err != nil { + it.Stop() + return nil, fmt.Errorf("%s's log %s: %w", l.Module, l.Name, err) + } + } + if !follow { + return it.Stop, nil + } + go func() { + defer it.Stop() + for { + m, err := it.Next() + if err != nil || (l.closed != nil && l.closed()) { + return + } + e, err := l.entry(m) + if err != nil || handle(e) != nil { + return + } + } + }() + return it.Stop, nil +} diff --git a/go/process/state_integration_test.go b/go/process/state_integration_test.go index 5c5f07f..7d86e70 100644 --- a/go/process/state_integration_test.go +++ b/go/process/state_integration_test.go @@ -274,6 +274,44 @@ func TestStateWatchOnAGrantedBucket(t *testing.T) { t.Fatalf("narrowed watch got %+v for a key it did not name", c) case <-time.After(500 * time.Millisecond): } + + // ADR 0571 clause 1: another module's log, read on the process's own credential with exactly the read grants: the + // newest per key, every entry from a time, and a watch from the newest that then hears an appended entry. + if _, err := adminJetStream.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_mesh-delivery-builder_build-log", + Subjects: []string{"mesh.log.mesh-delivery-builder.build-log.>"}, AllowDirect: true}); err != nil { + t.Fatal(err) + } + for _, k := range []string{"letta", "gitea", "letta"} { + if _, err := adminJetStream.Publish(ctx, "mesh.log.mesh-delivery-builder.build-log."+k, []byte(`{"module":"`+k+`"}`)); err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Millisecond) + } + builds := conn.ReadLog("mesh-delivery-builder", "build-log") + newest, err := builds.Newest("") + if err != nil || len(newest) != 2 || newest[0].Key != "gitea" || newest[1].Sequence != 3 { + t.Fatalf("the newest per key are %+v: %v", newest, err) + } + since, err := builds.Since(newest[0].At, "letta") + if err != nil || len(since) != 1 || since[0].Sequence != 3 { + t.Fatalf("letta since gitea's build is %+v: %v", since, err) + } + heard := make(chan stdio.LogEntry, 4) + if err := builds.Watch("letta", stdio.LogFromNewest, func(e stdio.LogEntry) error { heard <- e; return nil }); err != nil || len(heard) != 1 { + t.Fatalf("a watch from the newest handed %d at its start: %v", len(heard), err) + } + <-heard + if _, err := adminJetStream.Publish(ctx, "mesh.log.mesh-delivery-builder.build-log.letta", []byte(`{}`)); err != nil { + t.Fatal(err) + } + select { + case e := <-heard: + if e.Sequence != 4 || e.Key != "letta" { + t.Fatalf("the watch heard %+v", e) + } + case <-time.After(10 * time.Second): + t.Fatal("the watch did not hear the appended entry") + } } // 1840/AC8: given an account not granted a bucket, when it watches it, then the refusal names the bucket. diff --git a/go/stdio.go b/go/stdio.go index c3de051..82568f6 100644 --- a/go/stdio.go +++ b/go/stdio.go @@ -72,6 +72,8 @@ type channel struct { watches map[string]func(StateChange) error // provisioned are this bundle's watches of its values, by the id each was asked with (provision.go). provisioned map[string]func(ProvisionChange) error + // logWatches are this bundle's watches of a log, by the id each was asked with (log.go). + logWatches map[string]func(LogEntry) error // takers and provers are this bundle's seat work and proof handlers, by worker and subject (ADR 0259). takers map[string]func(Work) error provers map[string]func(subject string, body json.RawMessage) (any, error) @@ -182,8 +184,8 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error { byName[t.Name] = t } c := &channel{enc: json.NewEncoder(out), waiting: map[string]chan message{}, watches: map[string]func(StateChange) error{}, - provisioned: map[string]func(ProvisionChange) error{}, - takers: map[string]func(Work) error{}, provers: map[string]func(string, json.RawMessage) (any, error){}} + provisioned: map[string]func(ProvisionChange) error{}, logWatches: map[string]func(LogEntry) error{}, + takers: map[string]func(Work) error{}, provers: map[string]func(string, json.RawMessage) (any, error){}} currentMu.Lock() current = c currentMu.Unlock() @@ -315,6 +317,28 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error { } c.say(map[string]any{"id": id, "result": map[string]any{}}) }(id, *p.Change) + case "mesh/log": + // One entry of a log this bundle watches (log.go), named by the watch's id, answered once its handler has + // taken it, in order: the runtime sends the next only after this answer. + var p struct { + Watch string `json:"watch"` + Entry *LogEntry `json:"entry"` + } + _ = json.Unmarshal(m.Params, &p) + c.mu.Lock() + handle := c.logWatches[p.Watch] + c.mu.Unlock() + if handle == nil || p.Entry == nil { + c.say(map[string]any{"id": id, "error": map[string]any{"code": -32602, "message": "mesh/log names no watch of this bundle"}}) + continue + } + go func(id any, e LogEntry) { + if err := handle(e); err != nil { + c.say(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + c.say(map[string]any{"id": id, "result": map[string]any{}}) + }(id, *p.Entry) case "mesh/provision": // One change to a value of this module's that this bundle watches (provision.go), named by the // watch's id, answered once its handler has taken it. Beside the loop, like a tool call. From 08fc70783a7c2e37dde0ca7bb6faf5e63cfd4d10 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:31:34 +0100 Subject: [PATCH 42/52] Read the bus through a module's declared states in frombus, with the runtime's adapter in fromruntime, so mesh-provisioner and disk-load compose from one reader instead of two copies (issue 1955) --- go/compose/frombus/frombus_test.go | 81 +++++++++++ go/compose/frombus/states.go | 199 ++++++++++++++++++++++++++ go/compose/fromruntime/fromruntime.go | 49 +++++++ 3 files changed, 329 insertions(+) create mode 100644 go/compose/frombus/states.go create mode 100644 go/compose/fromruntime/fromruntime.go diff --git a/go/compose/frombus/frombus_test.go b/go/compose/frombus/frombus_test.go index 1a4b562..bb53330 100644 --- a/go/compose/frombus/frombus_test.go +++ b/go/compose/frombus/frombus_test.go @@ -14,6 +14,7 @@ import ( "os" "reflect" "slices" + "sort" "strings" "testing" @@ -724,6 +725,17 @@ func TestInput(t *testing.T) { } } } + // And the same through StatesReader, over the fake's states as a module's runtime gives them (one reader for + // mesh-provisioner and disk-load, the lead 2026-10-29). + viaStates, _, err := DataItems(ctx, StatesReader(statesOf(t, ctx, noSecrets), func(_ context.Context, url string) ([]byte, error) { + for _, body := range noSecrets.archives { + return body, nil + } + return nil, errors.New("no archive") + }), Mesh{}, Scope{Node: "n1"}) + if err != nil || !reflect.DeepEqual(viaStates, items) { + t.Errorf("through the states the data items are %v, %v; through the reader %v", viaStates, err, items) + } // And what a provider on another node serves from its settings (mailu's smtp domain, issue 1126: shanks refused // '${setting:domain} ... nothing sets "domain"'): read for that provider's served keys alone, through Served. @@ -769,3 +781,72 @@ func TestInput(t *testing.T) { } } } + +// mapStates are read states by name, key to value, as a module's runtime gives them. +type mapStates map[string]map[string][]byte + +func (m mapStates) Get(state, key string) ([]byte, bool, error) { + v, ok := m[state][key] + return v, ok, nil +} + +func (m mapStates) Keys(state string) ([]string, error) { + var out []string + for k := range m[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// statesOf is b laid out as the states StatesReader reads. +func statesOf(t *testing.T, ctx context.Context, b *bus) mapStates { + t.Helper() + out := mapStates{} + put := func(state, key string, raw []byte, err error) { + if err != nil { + t.Fatal(err) + } + if out[state] == nil { + out[state] = map[string][]byte{} + } + out[state][key] = raw + } + placements, _ := b.Placements(ctx) + for node, p := range placements { + k, _ := delivery.PlacementKey(node) + raw, err := delivery.EncodePlacement(p) + put(PlacementRead, k, raw, err) + } + records, _ := b.Records(ctx) + for node, r := range records { + k, _ := delivery.HaveNodeKey(node) + rec := r + raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &rec}) + put(HaveRead, k, raw, err) + } + modules, _ := b.Modules(ctx) + for name, m := range modules { + raw, err := delivery.EncodeModule(m) + put(ModulesRead, name, raw, err) + var mf struct{ Module string } + _ = json.Unmarshal(b.manifests[m.Manifest], &mf) + k, _ := delivery.ManifestsKey(m.Manifest) + raw, err = delivery.EncodeManifest(delivery.Manifest{Module: mf.Module, JSON: string(b.manifests[m.Manifest])}) + put(ManifestsRead, k, raw, err) + } + for key, build := range b.builds { + module, fp, _ := strings.Cut(key, " ") + k, _ := delivery.BuilderBuildKey(module, fp) + raw, err := delivery.EncodeBuilderBuild(*build) + put(BuildsRead, k, raw, err) + } + for nm, values := range b.values { + for k, v := range values { + value, _ := json.Marshal(v) + raw, err := json.Marshal(map[string]any{"value": json.RawMessage(value), "source": catalogue.SourceModule}) + put(ResolvedRead, nm+"."+k, raw, err) + } + } + return out +} diff --git a/go/compose/frombus/states.go b/go/compose/frombus/states.go new file mode 100644 index 0000000..9962cb9 --- /dev/null +++ b/go/compose/frombus/states.go @@ -0,0 +1,199 @@ +package frombus + +// **The bus read through a module's declared `reads`** (the lead, 2026-10-29: one reader in place of the copies in +// mesh-provisioner and disk-load): a Reader over the states a module reads, by a runtime's Get and Keys (a bundle's +// stdio.State, a process's own), for a holder that composes plans or data items and reads no value: placement, each +// node's record in its own have key, the indexer's modules and manifests, the builder's builds and mesh-config's resolved +// values. No own secret and no credential is read (compose.Planning). Pure as the rest of this package: a bundle's +// runtime states and an HTTP fetch are compose/fromruntime's. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The states a module names in its manifest's `reads` to compose through StatesReader. +const ( + PlacementRead = delivery.PlacementRead + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" + BuildsRead = delivery.BuilderModule + ".builds" + HaveRead = "node-engine.have" + ResolvedRead = "mesh-config.resolved" +) + +// StatesReads are those reads, in a manifest's order. +var StatesReads = []string{PlacementRead, ModulesRead, ManifestsRead, BuildsRead, HaveRead, ResolvedRead} + +// States is a module's read states as its runtime gives them: a key's value, false when it has none. +type States interface { + Get(state, key string) (value []byte, found bool, err error) + Keys(state string) ([]string, error) +} + +// Fetch reads an archive at a URL (fromruntime.HTTPFetch over HTTP). +type Fetch func(ctx context.Context, url string) ([]byte, error) + +// StatesReader is the Reader over s; fetch reads a templates archive, nil for a holder that reads none. +func StatesReader(s States, fetch Fetch) Reader { return statesReader{s: s, fetch: fetch} } + +type statesReader struct { + s States + fetch Fetch +} + +func (r statesReader) each(read string, visit func(key string, value []byte)) error { + keys, err := r.s.Keys(read) + if err != nil { + return fmt.Errorf("%s: %w", read, err) + } + for _, k := range keys { + v, found, err := r.s.Get(read, k) + if err != nil { + return fmt.Errorf("%s %s: %w", read, k, err) + } + if found { + visit(k, v) + } + } + return nil +} + +func (r statesReader) Placements(context.Context) (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + err := r.each(PlacementRead, func(k string, v []byte) { + if node, err := delivery.ParsePlacementKey(k); err == nil { + if p, err := delivery.DecodePlacement(v); err == nil && p.Node == node { + out[node] = p + } + } + }) + return out, err +} + +func (r statesReader) Records(context.Context) (map[string]delivery.NodeRecord, error) { + out := map[string]delivery.NodeRecord{} + err := r.each(HaveRead, func(k string, v []byte) { + if node, _, isNode, err := delivery.ParseHaveKey(k); err == nil && isNode { + if h, err := delivery.DecodeHaveNode(v); err == nil && h.Record != nil { + out[node] = *h.Record + } + } + }) + return out, err +} + +func (r statesReader) Modules(context.Context) (map[string]delivery.Module, error) { + out := map[string]delivery.Module{} + err := r.each(ModulesRead, func(k string, v []byte) { + if module, err := delivery.ParseModulesKey(k); err == nil { + if m, err := delivery.DecodeModule(v); err == nil { + out[module] = m + } + } + }) + return out, err +} + +func (r statesReader) Manifest(_ context.Context, digest string) ([]byte, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return nil, err + } + v, found, err := r.s.Get(ManifestsRead, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("the indexer holds no manifest %s", digest) + } + m, err := delivery.DecodeManifest(v) + if err != nil { + return nil, err + } + return []byte(m.JSON), nil +} + +func (r statesReader) Values(_ context.Context, node, module string) (map[string]any, error) { + var out map[string]any + prefix := node + "." + module + "." + err := r.each(ResolvedRead, func(k string, v []byte) { + name, ok := strings.CutPrefix(k, prefix) + if !ok || strings.Contains(name, ".") { + return + } + if value, laid := catalogue.LaidValue(v); laid { + if out == nil { + out = map[string]any{} + } + out[name] = value + } + }) + return out, err +} + +func (statesReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { + return nil, nil +} + +func (statesReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + return nil, nil, nil +} + +func (r statesReader) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return nil, err + } + v, found, err := r.s.Get(BuildsRead, key) + if err != nil || !found { + return nil, err + } + b, err := delivery.DecodeBuilderBuild(v) + if err != nil { + return nil, err + } + return &b, nil +} + +// archiveWithin bounds one read of a templates archive. +const archiveWithin = 30 * time.Second + +func (r statesReader) Archive(ctx context.Context, url, digest string) ([]byte, error) { + if r.fetch == nil { + return nil, fmt.Errorf("this holder reads no archive (%s)", url) + } + fetching, cancel := context.WithTimeout(ctx, archiveWithin) + defer cancel() + body, err := r.fetch(fetching, url) + if err != nil { + return nil, err + } + sum := sha256.Sum256(body) + if got := hex.EncodeToString(sum[:]); got != strings.TrimPrefix(digest, "sha256:") { + return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got) + } + return body, nil +} + +func (r statesReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) { + all, err := r.Values(ctx, node, module) + var out map[string]any + for _, k := range keys { + if v, ok := all[k]; ok { + if out == nil { + out = map[string]any{} + } + out[k] = v + } + } + return out, err +} diff --git a/go/compose/fromruntime/fromruntime.go b/go/compose/fromruntime/fromruntime.go new file mode 100644 index 0000000..a2a98e0 --- /dev/null +++ b/go/compose/fromruntime/fromruntime.go @@ -0,0 +1,49 @@ +// Package fromruntime is compose/frombus's StatesReader over a bundle's runtime (the lead, 2026-10-29): the states a +// module's manifest names in `reads` (frombus.StatesReads), read through stdio.State, and a templates archive fetched +// over HTTP. Apart from frombus so that frombus stays pure for the mesh-compose command. +package fromruntime + +import ( + "context" + "fmt" + "io" + "net/http" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" +) + +// States are a bundle's read states through its runtime. +type States struct{} + +// Get is a key's value, false when it has none. +func (States) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// Keys are a state's keys. +func (States) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// Reader is frombus's Reader over this bundle's read states, fetching archives over HTTP. +func Reader() frombus.Reader { return frombus.StatesReader(States{}, HTTPFetch) } + +// HTTPFetch reads an archive over HTTP, at most 256 MiB. +func HTTPFetch(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s", url, resp.Status) + } + return io.ReadAll(io.LimitReader(resp.Body, 256<<20)) +} From 9f0bb1cfa4ba56a971d652faaaa59cc9bdd684e9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:37:37 +0100 Subject: [PATCH 43/52] Move restic's last good night per item into the sdk's health layouts, so restic's night process writes it and the self-check reads one layout (issue 1955, ADR 0571 clause 3) --- go/buscompose/buscompose_test.go | 13 ++++++++--- go/health/condition_test.go | 9 ++++++++ go/health/readings.go | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index a83b771..0e37746 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -614,7 +614,7 @@ func healthChecks(t *testing.T) { users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": { {Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")}, Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, - {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}}, + {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good"}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}}, {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, @@ -630,7 +630,7 @@ func healthChecks(t *testing.T) { {Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}}, // ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them. {Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}}, - {Module: "restic", Reads: []string{"disk-load.readings"}}, + {Module: "restic", Reads: []string{"disk-load.readings"}, Processes: []ProcessPrincipal{{Process: "night", State: []string{"good"}}}}, // ADR 0571 clauses 1, 9: the indexer reads the builder's log through its own consumer; the catalogue module asks // the builder's tool. {Module: delivery.BuilderModule, Logs: []Log{{Module: delivery.BuilderModule, Name: delivery.BuilderLog, MaxAgeDays: 30}}}, @@ -652,6 +652,9 @@ func healthChecks(t *testing.T) { "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, + // ADR 0571 clause 3: restic's night writes each item's last good night; the self-check reads it. + "a.restic.night": {"$KV.restic_good.a.pg.data"}, + "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"}, "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", // ADR 0571 clause 7: its own membership from the grants part's state. "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"}, @@ -670,12 +673,16 @@ func healthChecks(t *testing.T) { mayNot := map[string][]string{ // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, - "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch"}, + "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data"}, // Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570 // clauses 1, 8). "a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"}, "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", "mesh.seat.artifact-store.event.artifacts-gone"}, + // Each writes its own state alone, and these users are composed. + "a.restic.night": {"$KV.disk-load_readings.a.pg.data"}, + "a.disk-load": {"$KV.restic_good.a.pg.data"}, + "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, // Only the assigner writes seat-holders. "a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"}, // Only disk-load writes its readings. diff --git a/go/health/condition_test.go b/go/health/condition_test.go index 9c25aab..348e62a 100644 --- a/go/health/condition_test.go +++ b/go/health/condition_test.go @@ -92,6 +92,15 @@ func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" { t.Fatal("the readings' key or bucket moved") } + // And restic's last good night per item, keyed as a reading. + goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at}) + if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" || + GoodBucket != "restic_good" || GoodRead != "restic.good" { + t.Fatalf("a last good night read back as %+v: %v", g, err) + } + if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil { + t.Fatal("a last good night of another version was read") + } good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing", Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} diff --git a/go/health/readings.go b/go/health/readings.go index a0983c9..2686259 100644 --- a/go/health/readings.go +++ b/go/health/readings.go @@ -125,3 +125,41 @@ func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Meas a.MeasuredAt, b.MeasuredAt = nil, nil return r, !reflect.DeepEqual(a, b) } + +// **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder +// restic keeps, from its night's process, its own state `good`, one key `..` for every item a module +// declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest +// good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here +// from mesh-catalog modules/restic (e5ef3bed). +const ( + ResticModule = "restic" + GoodState = "good" + GoodBucket = ResticModule + "_" + GoodState + GoodRead = ResticModule + "." + GoodState + GoodVersion = 1 +) + +// GoodRow is one key of restic's `good`. +type GoodRow struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Item string `json:"item"` + CoveredBy string `json:"covered_by"` + LastGood time.Time `json:"last_good,omitzero"` +} + +// GoodKey is a row's key: `..`, as a reading's. +func GoodKey(node, module, item string) string { return ReadingKey(node, module, item) } + +// DecodeGoodRow reads a row; it refuses another version. +func DecodeGoodRow(raw []byte) (GoodRow, error) { + var g GoodRow + if err := json.Unmarshal(raw, &g); err != nil { + return GoodRow{}, fmt.Errorf("health: a last good night is not readable: %w", err) + } + if g.V != GoodVersion { + return GoodRow{}, fmt.Errorf("health: a last good night of version %d, and this reads %d", g.V, GoodVersion) + } + return g, nil +} From 85f5fd0b2266940f6652cbc66e140ef8b9b8d404 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:38:59 +0100 Subject: [PATCH 44/52] Fail the grants table test for any row whose user is not composed, so a row can never pass for a missing user (issue 1955) --- go/buscompose/buscompose_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 0e37746..ebceb40 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -800,9 +800,12 @@ func healthChecks(t *testing.T) { } } } - for name := range mayNot { - if !seen[name] { - t.Errorf("no user %s was composed", name) + // Every row names a user that was composed, in either table: a row for a user absent would pass for nothing. + for _, table := range []map[string][]string{may, mayNot} { + for name := range table { + if !seen[name] { + t.Errorf("no user %s was composed", name) + } } } } From 992ea0ab6695f9931edeb1fa62c484dc2ffd52dc Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:41:16 +0100 Subject: [PATCH 45/52] =?UTF-8?q?Let=20restic's=20bundle,=20not=20its=20ni?= =?UTF-8?q?ght's=20process,=20write=20each=20item's=20last=20good=20night,?= =?UTF-8?q?=20since=20the=20night=20runs=20as=20the=20operator's=20account?= =?UTF-8?q?=20with=20no=20bus=20account=20(issue=201955,=20ADR=200436=20?= =?UTF-8?q?=C2=A71-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go/buscompose/buscompose_test.go | 8 +++----- go/health/readings.go | 3 ++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index ebceb40..290a569 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -630,7 +630,7 @@ func healthChecks(t *testing.T) { {Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}}, // ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them. {Module: "disk-load", State: []Bucket{bucket("disk-load", "readings")}}, - {Module: "restic", Reads: []string{"disk-load.readings"}, Processes: []ProcessPrincipal{{Process: "night", State: []string{"good"}}}}, + {Module: "restic", Reads: []string{"disk-load.readings"}, State: []Bucket{bucket("restic", "good")}}, // ADR 0571 clauses 1, 9: the indexer reads the builder's log through its own consumer; the catalogue module asks // the builder's tool. {Module: delivery.BuilderModule, Logs: []Log{{Module: delivery.BuilderModule, Name: delivery.BuilderLog, MaxAgeDays: 30}}}, @@ -652,10 +652,9 @@ func healthChecks(t *testing.T) { "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, - // ADR 0571 clause 3: restic's night writes each item's last good night; the self-check reads it. - "a.restic.night": {"$KV.restic_good.a.pg.data"}, + // ADR 0571 clause 3: restic's bundle writes each item's last good night as its own state; the self-check reads it. "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"}, - "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", + "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", "$KV.restic_good.a.pg.data", // ADR 0571 clause 7: its own membership from the grants part's state. "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"}, "a.mesh-delivery-indexer": {"$JS.API.CONSUMER.CREATE.LOG_mesh-delivery-builder_build-log.r1.mesh.log.mesh-delivery-builder.build-log.letta", @@ -680,7 +679,6 @@ func healthChecks(t *testing.T) { "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", "mesh.seat.artifact-store.event.artifacts-gone"}, // Each writes its own state alone, and these users are composed. - "a.restic.night": {"$KV.disk-load_readings.a.pg.data"}, "a.disk-load": {"$KV.restic_good.a.pg.data"}, "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, // Only the assigner writes seat-holders. diff --git a/go/health/readings.go b/go/health/readings.go index 2686259..9b8d189 100644 --- a/go/health/readings.go +++ b/go/health/readings.go @@ -127,7 +127,8 @@ func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Meas } // **Each item's last good night** (novox/hq ADR 0571 clause 3, ADR 0564 §6; the lead, 2026-10-29): the node-backup holder -// restic keeps, from its night's process, its own state `good`, one key `..` for every item a module +// restic keeps, from its tools bundle through the runtime on the night's file event (its night's process runs as the +// operator's account and holds no bus account, ADR 0436 §1-2), its own state `good`, one key `..` for every item a module // declares that a backup path covers, written when it changes: the item, the path whose snapshot keeps it and the newest // good night of its module (zero before the first). The self-check's probe of a stale backup reads it there. Moved here // from mesh-catalog modules/restic (e5ef3bed). From f3677eb6f36e19ba1aec7c87001490bca49fd231 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:48:08 +0100 Subject: [PATCH 46/52] Carry the engine's verdict on each account declared never to become root, and its contract, in the node's health report, so the self-check's DA and D-root judge on the machine's own word (issue 1952, ADR 0266) NodeHealth.Root: each such account's verdict (healthy, unhealthy with its way to root, unknown while the setuid search runs), the module and resource declaring it. NodeHealth.Contract: the engine's statement contract, so a reader tells an engine older than the judging. Test: a row in TestTheControlRecordsReadTheirWritersValues, read back byte-equal. --- go/delivery/control_records_test.go | 4 ++++ go/delivery/nodehealth.go | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/go/delivery/control_records_test.go b/go/delivery/control_records_test.go index 2c64037..51aed3c 100644 --- a/go/delivery/control_records_test.go +++ b/go/delivery/control_records_test.go @@ -32,6 +32,10 @@ func TestTheControlRecordsReadTheirWritersValues(t *testing.T) { `"relogin":[{"account":"jochen","module":"brother","resource":"brother.scanner","reason":"relogin needed: the session began before scanner","since":"2026-10-29T20:00:00Z"}],` + `"found":[{"module":"photos","resource":"photos.library","path":"/srv/photos","reason":"used as found: owned by root","since":"2026-10-29T20:00:00Z"}],` + `"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}}, + // The engine's verdict on an account declared never to become root (ADR 0266), and its contract. + {"a node's health report, the agent account judged", `{"v":1,"node":"anchor","root":[{"account":"agent",` + + `"module":"claude-code","resource":"claude-code.agent","state":"healthy","since":"2026-10-29T20:00:00Z"}],` + + `"contract":13,"at":"2026-10-29T20:00:00Z"}`, &NodeHealth{}}, // A part pointing at another machine names it (issue 774): the hub's tunnel, held at the hub's machine. {"a node's health report, a part pointing at the hub", `{"v":1,"node":"anchor","network":{"state":"unhealthy",` + `"since":"2026-10-29T20:00:00Z","parts":[{"part":"tunnel","state":"unhealthy","reason":"the tunnel to the hub has not handshaken for over five minutes",` + diff --git a/go/delivery/nodehealth.go b/go/delivery/nodehealth.go index b9928a3..62c541a 100644 --- a/go/delivery/nodehealth.go +++ b/go/delivery/nodehealth.go @@ -40,6 +40,13 @@ type NodeHealth struct { Relogin []NodeAccount `json:"relogin,omitempty"` // Found are the directories used as found. Found []NodeDirectory `json:"found,omitempty"` + // Root are the engine's verdicts on each account a module declares never to become root without a person (novox/hq + // ADR 0266), sorted: the agent account's among them where a module declares it. What the self-check's DA and D-root + // judge an agent's confinement by, on the machine's own word. + Root []RootVerdict `json:"root,omitempty"` + // Contract is the engine's statement contract (mesh-host link's NodeKeyContract): a reader tells an engine older + // than a judging it reads (RootContract for Root) from one that judged nothing wrong. + Contract int `json:"contract,omitempty"` // At is when the engine wrote this, on a change: never a reason to write. At time.Time `json:"at"` } @@ -113,6 +120,21 @@ type NodeAccount struct { Since time.Time `json:"since"` } +// RootContract is the statement contract from which an engine judges an account's root (mesh-host link.RootContract). +const RootContract = 3 + +// RootVerdict is the engine's verdict on one account declared never to become root without a person: healthy (no way +// found), unhealthy (its reason starts "can become root without a person"), or unknown while its setuid search has not +// completed, with the module and resource declaring it. +type RootVerdict struct { + Account string `json:"account"` + Module string `json:"module"` + Resource string `json:"resource"` + State string `json:"state"` + Reason string `json:"reason,omitempty"` + Since time.Time `json:"since"` +} + // NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why. type NodeDirectory struct { Module string `json:"module"` From 7bd0f6c2bdf2f85678d3957764aa8e0f139bbb37 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:53:51 +0100 Subject: [PATCH 47/52] Name the artifact sweep's seat mesh-artifact-sweep for its scope, grant the provisioner client its machine's plans in place of the needs, and keep the grants part's withheld hearings in one layout, so the seat names follow ADR 0121 and the client and the self-check read the bus (issue 1955, ADR 0571 clause 2, ADR 0570 clause 2) --- go/buscompose/buscompose_test.go | 25 ++++++++++++---- go/buscompose/provisioner.go | 16 ++++++---- go/buscompose/userlist_switch.go | 29 ++++++++++++++++++- ...t_store_seat.go => artifact_sweep_seat.go} | 27 ++++++++--------- go/catalogue/seats.go | 6 ++-- go/catalogue/seats_test.go | 4 +-- 6 files changed, 74 insertions(+), 33 deletions(-) rename go/catalogue/{artifact_store_seat.go => artifact_sweep_seat.go} (70%) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 290a569..0138721 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -614,7 +614,9 @@ func healthChecks(t *testing.T) { users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": { {Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")}, Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, - {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good"}}, + {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good", GrantsWithheldRead}}, + // Issue 1955 L7: the grants part keeps the hearings it withholds; the self-check reads them. + {Module: GrantsModule, State: []Bucket{bucket(GrantsModule, GrantsWithheldState)}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, {Module: WatchmanModule, State: []Bucket{bucket(WatchmanModule, "findings")}, Reads: []string{delivery.NodeHealthRead}}, {Module: "mesh-hand-acts", State: []Bucket{bucket("mesh-hand-acts", "warrants")}, Logs: []Log{{Module: "mesh-hand-acts", Name: "acts"}}}, @@ -624,8 +626,10 @@ func healthChecks(t *testing.T) { bucket("mesh-provisioner", "registry")}}, {Module: "mesh-vault", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}}, {Module: "postgres", Reads: []string{"mesh-provisioner.provisions", "mesh-provisioner.plans"}}, - // ADR 0571 clauses 2, 4 (issue 1955): the artifact store's seat says artifacts-gone; the assigner keeps seat-holders. - {Module: "artifact-store-tools", Holds: []Seat{{Name: "artifact-store", Emits: []string{"artifacts-gone"}}}, + // The node's provisioner client reads its machine's plans and records no needs (ADR 0570 clause 2; its V3). + {Module: "mesh-provisioner-client", ProvisionsNode: true}, + // ADR 0571 clauses 2, 4 (issue 1955): the seat mesh-artifact-sweep says artifacts-gone; the assigner keeps seat-holders. + {Module: "artifact-store-tools", Holds: []Seat{{Name: "mesh-artifact-sweep", Emits: []string{"artifacts-gone"}}}, Reads: []string{delivery.SeatHoldersRead}}, {Module: delivery.AssignerModule, State: []Bucket{bucket(delivery.AssignerModule, delivery.SeatHoldersState)}}, // ADR 0571 clause 3: disk-load keeps the readings; the backup holder reads them. @@ -652,8 +656,12 @@ func healthChecks(t *testing.T) { "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak"}, "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, + "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak", + "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.c1.$KV.mesh-provisioner_plans.a.keycloak"}, // ADR 0571 clause 3: restic's bundle writes each item's last good night as its own state; the self-check reads it. - "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"}, + "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data", + "$JS.API.DIRECT.GET.KV_mesh-grants_withheld.$KV.mesh-grants_withheld." + WithheldKey(Withheld{Node: "a", Module: "reader", Stream: "MAIL"})}, + "a.mesh-grants": {"$KV.mesh-grants_withheld.reader.mail.a"}, "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", "$KV.restic_good.a.pg.data", // ADR 0571 clause 7: its own membership from the grants part's state. "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"}, @@ -672,15 +680,20 @@ func healthChecks(t *testing.T) { mayNot := map[string][]string{ // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, - "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data"}, + "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data", + "$KV.mesh-grants_withheld.reader.mail.a"}, // Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570 // clauses 1, 8). "a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"}, "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", - "mesh.seat.artifact-store.event.artifacts-gone"}, + "mesh.seat.mesh-artifact-sweep.event.artifacts-gone"}, // Each writes its own state alone, and these users are composed. "a.disk-load": {"$KV.restic_good.a.pg.data"}, "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, + // Another machine's plans are not its to read; no client writes or reads the needs any more. + "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.b.keycloak", + "$KV.mesh-provisioner_plans.a.keycloak", "$KV." + ProvisionerNeedsBucket + ".a.keycloak.db.x", + "$JS.API.DIRECT.GET.KV_" + ProvisionerNeedsBucket + ".$KV." + ProvisionerNeedsBucket + ".a.keycloak.db.x"}, // Only the assigner writes seat-holders. "a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"}, // Only disk-load writes its readings. diff --git a/go/buscompose/provisioner.go b/go/buscompose/provisioner.go index 5def5fd..19cd361 100644 --- a/go/buscompose/provisioner.go +++ b/go/buscompose/provisioner.go @@ -3,9 +3,11 @@ package buscompose import ( "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery" "strings" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" + "github.com/nats-io/nats.go" ) @@ -34,9 +36,10 @@ var ConfigResolvedBucket = BucketName("mesh-config", "resolved") // ProvisionerReads are the domains' buckets a provisioner client reads its own machine's keys of, each keyed // `.…` (research 051): placement, the controller's assignments mirror (`.`), whose value carries // the provisioner's plan of a module handed to it (inventory.Provisioning, design 57 P5); and config, -// mesh-config's resolved values. **Named one by one**, as the controller's asks are: a domain that moves to the +// mesh-config's resolved values; and the seat mesh-provisioner's plans of its machine, `.` (novox/hq ADR +// 0570 clause 2). **Named one by one**, as the controller's asks are: a domain that moves to the // provisioner adds its bucket here, and a bucket not named is one the client cannot read. -var ProvisionerReads = []string{ConfigResolvedBucket} +var ProvisionerReads = []string{ConfigResolvedBucket, provisions.PlansBucket} // ProvisionerReadsWhole are the buckets a provisioner client reads every key of: the controller's assignments mirror, // where a provider's consumers are bound to it in their own machines' plans, which the provider machine's client @@ -68,9 +71,10 @@ var ( ) // ProvisionerWrites are the server's buckets a client writes, and reads, its own machine's keys of (`.>`) -// and no other's: the needs it records, what it applied, and the items of the collections on its machine. A -// client cannot write another machine's needs, or say another machine applied a file. -var ProvisionerWrites = []string{ProvisionerNeedsBucket, ProvisionerAppliedBucket, ProvisionerCollectBucket} +// and no other's: what it applied, and the items of the collections on its machine. A client cannot say another +// machine applied a file. It records no needs: since the client reads its machine's plans from the seat +// mesh-provisioner's `plans` (novox/hq ADR 0570 clause 2; the client's V3), the needs bucket is granted to none. +var ProvisionerWrites = []string{ProvisionerAppliedBucket, ProvisionerCollectBucket} // VerbsTheProvisionerAsksTheVault are the vault's verbs a provisioner client asks in its own name (ADR 0349 // clause 11): `get`, a secret its machine's declaration names, answered sealed to the asker's key. On the subject diff --git a/go/buscompose/userlist_switch.go b/go/buscompose/userlist_switch.go index 1bc1303..dc187d8 100644 --- a/go/buscompose/userlist_switch.go +++ b/go/buscompose/userlist_switch.go @@ -3,6 +3,7 @@ package buscompose import ( "slices" "strings" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/handacts" @@ -79,7 +80,7 @@ func grantsOwnObjects(p Principal, pattern string) bool { // readers too (announce.State.Readers); both are honoured. const VaultBusAccounts = "mesh-vault.bus-accounts" -// ArtifactStoreTools is the module holding the seat artifact-store (novox/hq ADR 0571 clause 2). +// ArtifactStoreTools is the module holding the seat mesh-artifact-sweep (novox/hq ADR 0571 clause 2). const ArtifactStoreTools = "artifact-store-tools" // NamedReaders are the states only the modules named may read, by `.`: the grants part's wanted accounts @@ -125,3 +126,29 @@ func withNamedReaders(d Declared) Declared { d.ReadsRefused = append(append([]string(nil), d.ReadsRefused...), refused...) return d } + +// **The hearings the grants part withholds, on the bus** (issue 1955 L7, mesh-delivery e564eaf): a hearing of a restricted +// stream a module asks in the tool runner, withheld because only an account of its own may hear it (ADR 0376 §2), kept +// in the grants part's state `withheld`, one key `..` (the controller's condition id +// of it), written on change only and deleted once heard. The self-check's probe D-heard reads it there. +const ( + GrantsWithheldState = "withheld" + GrantsWithheldBucket = GrantsModule + "_" + GrantsWithheldState + GrantsWithheldRead = GrantsModule + "." + GrantsWithheldState + WithheldVersion = 1 +) + +// WithheldHearing is one withheld hearing: the machine, the module, the restricted stream and its emitter, the plain +// words of it and the remedy (Withheld.Said), and when the grants part first wrote it as it stands. +type WithheldHearing struct { + V int `json:"v"` + Node string `json:"node"` + Module string `json:"module"` + Stream string `json:"stream"` + Emitter string `json:"emitter"` + Said string `json:"said"` + At time.Time `json:"at"` +} + +// WithheldKey is a withheld hearing's key: `..`. +func WithheldKey(w Withheld) string { return w.Module + "." + strings.ToLower(w.Stream) + "." + w.Node } diff --git a/go/catalogue/artifact_store_seat.go b/go/catalogue/artifact_sweep_seat.go similarity index 70% rename from go/catalogue/artifact_store_seat.go rename to go/catalogue/artifact_sweep_seat.go index 3fa1bf6..cf8cefc 100644 --- a/go/catalogue/artifact_store_seat.go +++ b/go/catalogue/artifact_sweep_seat.go @@ -1,16 +1,17 @@ package catalogue -// **The artifact store's sweep, out of the controller** (novox/hq ADR 0571 clause 2, issue 1955): the seat -// `artifact-store`, held once for the mesh by artifact-store-tools, serves what the controller served of its store -// (artifacts, held, mirrors, images, collect) and says `artifacts-gone`. Its keep set is the store's own listing joined +// **The artifact sweep, out of the controller** (novox/hq ADR 0571 clause 2, issue 1955): what the artifact store keeps +// and lets go of, the seat `mesh-artifact-sweep` (named for its scope, ADR 0121 §1; the operator, 2026-10-29), held +// once for the mesh by artifact-store-tools, serves what the controller served of its store (artifacts, held, mirrors, images, collect) and says `artifacts-gone`. Its keep set is the store's own listing joined // with every want's fingerprint and each module's five newest builds from the builder's log (ADR 0526 §1); it lets go // of nothing until the controller's lease names a holder that is no controller. -// ArtifactStoreSeat is the seat artifact-store-tools holds. -const ArtifactStoreSeat = "artifact-store" +// ArtifactSweepSeat is the seat artifact-store-tools holds. Not the seat mesh-artifact-store, the store itself held by +// distribution, and not the provision artifact-store it delivers. +const ArtifactSweepSeat = "mesh-artifact-sweep" -// artifactStoreVerbs are the seat's verbs, with the controller's arguments so a caller changes only the seat it asks. -func artifactStoreVerbs() []Verb { +// artifactSweepVerbs are the seat's verbs, with the controller's arguments so a caller changes only the seat it asks. +func artifactSweepVerbs() []Verb { return []Verb{ {Name: "artifacts", Description: "Every artifact the mesh made that lives in its artifact store, each with its " + "repository, digest, kind (image or archive) and state: kept, and why (a want names it, or one of the five " + @@ -37,14 +38,10 @@ func artifactStoreVerbs() []Verb { "why": "with confirm: why, required and recorded", "most": "let go of at most this many (500 by default, at most 5000)", }, nil, "confirm")}, - {Name: "mirrors", Description: "The bases the mesh copied into its artifact store for builds to stand on, each " + - "with its state: kept because a kept build stood on it, or eligible. With record and confirm and why, copies " + - "no record names are recorded as the mesh's. Deletes nothing.", - Input: schema(map[string]string{ - "record": "references to record, artifact-store://@sha256:, separated by spaces or commas", - "confirm": "\"true\": record them (needs why); without it nothing is recorded", - "why": "with confirm: why, required and recorded", - }, nil, "confirm")}, + {Name: "mirrors", Description: "The bases the mesh copied into its artifact store, each kept (a kept build stood on " + + "it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood on " + + "it: never removed), by what the builder's log says each build stood on. Records nothing by hand.", + Input: schema(map[string]string{}, nil)}, {Name: "images", Description: "Every container image one machine's want and have name, each with the resources " + "naming it: what the machine keeps when it prunes its images. Reads only.", Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})}, diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index b45ff1e..e8e68b0 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -303,9 +303,9 @@ var defaultSeats = append([]Seat{ // policy and the bus step at the assigner, and the hand-act log. {Name: WantsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"policy-changed"}, Serves: wantsVerbs()}, {Name: HandActsSeat, Scope: ScopeMesh, Decision: controlRecordsDecision, Emits: []string{"act-recorded"}, Serves: handActsVerbs()}, - // The artifact store's sweep (novox/hq ADR 0571 clause 2, artifact_store_seat.go). - {Name: ArtifactStoreSeat, Scope: ScopeMesh, Decision: "novox/hq ADR 0571", Emits: []string{"artifacts-gone"}, - Serves: artifactStoreVerbs()}, + // The artifact sweep (novox/hq ADR 0571 clause 2, artifact_sweep_seat.go). + {Name: ArtifactSweepSeat, Scope: ScopeMesh, Decision: "novox/hq ADR 0571", Emits: []string{"artifacts-gone"}, + Serves: artifactSweepVerbs()}, // Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a // delivering-seat migration with a mesh-wide cascade if a holder stops resolving mid-flight. // They keep their names until that migration is done deliberately, apart from the node-* pass. diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index 3182411..e6b206b 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -77,7 +77,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { {WantsSeat, "upgrade,bus", "policy-changed"}, {HandActsSeat, "hand-act,hand-acts,warranted", "act-recorded"}, // ADR 0571 clause 2: the artifact store's sweep. - {ArtifactStoreSeat, "artifacts,held,collect,mirrors,images", "artifacts-gone"}, + {ArtifactSweepSeat, "artifacts,held,collect,mirrors,images", "artifacts-gone"}, } { s, ok := SeatNamed(c.seat) if !ok || s.Scope != ScopeMesh || s.Replicated || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 05") || @@ -122,7 +122,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { // mesh-build-machine row goes, when no registered manifest claims it. // Forty-eight with mesh-delivery-checker (novox/hq ADR 0547 §10, issue 774); forty-nine with node-engine (novox/hq ADR 0563 §1); // fifty-three with the four health seats (novox/hq ADR 0564 clause 1); fifty-five with mesh-wants and mesh-hand-acts - // (novox/hq ADR 0565 clause 1); fifty-six with artifact-store (novox/hq ADR 0571 clause 2). + // (novox/hq ADR 0565 clause 1); fifty-six with mesh-artifact-sweep (novox/hq ADR 0571 clause 2). if len(Seats()) != 56 { t.Errorf("the mesh defines %d seats rather than 56; the set is closed, so a change here is "+ "a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames()) From 31e90aaa0f0d7749bcbff2d5d7db649fdd2ba967 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:56:06 +0100 Subject: [PATCH 48/52] Keep when each data item was first seen and when and why it was retired in its reading, so the self-check's D13 judges a new or retired item from disk-load's state (issue 1955, ADR 0571 clause 3) --- go/health/condition_test.go | 16 ++++++++++++++++ go/health/readings.go | 12 +++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/go/health/condition_test.go b/go/health/condition_test.go index 348e62a..f0085b7 100644 --- a/go/health/condition_test.go +++ b/go/health/condition_test.go @@ -89,6 +89,22 @@ func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { } prev = &r } + // The first reading sets FirstSeen and every one after keeps it, as it keeps a retirement; an older reading reads as before. + if first := time.Date(2026, 10, 20, 1, 0, 0, 0, time.UTC); !prev.FirstSeen.Equal(first) { + t.Fatalf("first seen %s, not %s", prev.FirstSeen, first) + } + retired := *prev + retired.RetiredAt, retired.RetiredWhy = at, "its module left ace" + if r, _ := NextReading(&retired, "ace", retired.PlacedDataItem, retired.Measured, at.Add(time.Hour)); !r.RetiredAt.Equal(at) || + r.RetiredWhy != "its module left ace" || !r.FirstSeen.Equal(prev.FirstSeen) { + t.Fatalf("a retired reading moved on as %+v", r) + } + if old, err := DecodeReading([]byte(`{"v":1,"node":"ace","module":"pg","item":"data"}`)); err != nil || !old.FirstSeen.IsZero() { + t.Fatalf("a reading of before read as %+v: %v", old, err) + } + if raw, _ := json.Marshal(Reading{V: ReadingVersion}); strings.Contains(string(raw), "first_seen") || strings.Contains(string(raw), "retired") { + t.Fatalf("zero fields were written: %s", raw) + } if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" { t.Fatal("the readings' key or bucket moved") } diff --git a/go/health/readings.go b/go/health/readings.go index 9b8d189..e02c960 100644 --- a/go/health/readings.go +++ b/go/health/readings.go @@ -67,6 +67,12 @@ type Reading struct { Peak *int64 `json:"peak_bytes,omitempty"` PeakAt string `json:"peak_day,omitempty"` Days []DaySize `json:"days,omitempty"` + // FirstSeen is when the item was first measured on this node, kept by every reading after; RetiredAt and RetiredWhy + // say its module left the node, written by whoever retires it and carried on unchanged. Each is left out while zero, + // so a reading written before reads as it did. + FirstSeen time.Time `json:"first_seen,omitzero"` + RetiredAt time.Time `json:"retired_at,omitzero"` + RetiredWhy string `json:"retired_why,omitempty"` } // ReadingKey is the key of an item on a node: `..`. @@ -88,10 +94,14 @@ func DecodeReading(raw []byte) (Reading, error) { // was taken: a reading that did not change is not written. A measurement with no size (an error, a shallow item) carries // the last size's days on, so the peak outlives one failed measurement. func NextReading(prev *Reading, node string, it catalogue.PlacedDataItem, m Measured, now time.Time) (Reading, bool) { - r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m} + r := Reading{V: ReadingVersion, Node: node, PlacedDataItem: it, Measured: m, FirstSeen: now.UTC()} var days []DaySize if prev != nil { days = prev.Days + r.RetiredAt, r.RetiredWhy = prev.RetiredAt, prev.RetiredWhy + if !prev.FirstSeen.IsZero() { + r.FirstSeen = prev.FirstSeen + } } today := now.UTC().Format("2006-01-02") cut := now.UTC().Add(-PeakWindow).Format("2006-01-02") From fa5a6aeff51334395e897a880b1ca884464f9d3d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:58:21 +0100 Subject: [PATCH 49/52] Let a bundle ask the runtime who answers a verb, and the self-check's own account ask the bus's discovery, so D-root and D3 find holders without the controller (issue 1955) --- go/buscompose/buscompose_test.go | 10 +++++-- go/buscompose/health_grants.go | 3 ++ go/buscompose/nats.go | 5 ++++ go/discover.go | 48 ++++++++++++++++++++++++++++++++ go/log_test.go | 25 +++++++++++++++++ 5 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 go/discover.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 0138721..bd56f70 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -41,6 +41,12 @@ var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATE // write. func beyondTheControllers(user string) map[string]bool { out := map[string]bool{} + // The self-check's runtime asks who answers (D-root, D3). + if strings.HasSuffix(user, "."+SelfCheckModule) { + for _, s := range discovering() { + out[s] = true + } + } if strings.HasSuffix(user, "."+WatchmanModule) { for _, s := range append(WatchmanDeadLetterGrants(nil), WatchmanHears()...) { out[s] = true @@ -659,7 +665,7 @@ func healthChecks(t *testing.T) { "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.c1.$KV.mesh-provisioner_plans.a.keycloak"}, // ADR 0571 clause 3: restic's bundle writes each item's last good night as its own state; the self-check reads it. - "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data", + "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data", "$SRV.INFO.node-tools", "$SRV.PING", "$JS.API.DIRECT.GET.KV_mesh-grants_withheld.$KV.mesh-grants_withheld." + WithheldKey(Withheld{Node: "a", Module: "reader", Stream: "MAIL"})}, "a.mesh-grants": {"$KV.mesh-grants_withheld.reader.mail.a"}, "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", "$KV.restic_good.a.pg.data", @@ -688,7 +694,7 @@ func healthChecks(t *testing.T) { "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", "mesh.seat.mesh-artifact-sweep.event.artifacts-gone"}, // Each writes its own state alone, and these users are composed. - "a.disk-load": {"$KV.restic_good.a.pg.data"}, + "a.disk-load": {"$KV.restic_good.a.pg.data", "$SRV.INFO", "$SRV.PING.node-tools"}, "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, // Another machine's plans are not its to read; no client writes or reads the needs any more. "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.b.keycloak", diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go index e31a935..916853d 100644 --- a/go/buscompose/health_grants.go +++ b/go/buscompose/health_grants.go @@ -170,3 +170,6 @@ func keepsDeadLetters(p Principal, pattern string) bool { return isController(p, pattern) || (p.Kind == KindModule && p.Module == WatchmanModule && pattern == deadLetterPrefix+">") } + +// SelfCheckModule is the module holding the seat mesh-self-check, whose runtime asks the bus who answers (D-root, D3). +const SelfCheckModule = health.SelfCheckModule diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 61bdfa9..64b6adf 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -1014,6 +1014,11 @@ func PermissionsFor(p Principal) (Permissions, error) { // And the words its watchdogs hear (ADR 0564 clause 4, Limits 3, 11). sub = append(sub, WatchmanHears()...) } + // The self-check asks who answers a verb (D-root, D3): its runtime, on the self-check's own account (runs-as), + // asks the bus's discovery, its replies to its own inbox; no other module's account may. + if p.Module == SelfCheckModule { + pub = append(pub, discovering()...) + } // The grants part alone publishes the modules' memberships into the assignments stream (novox/hq ADR 0547 §5; // the operator, 2026-10-25 22:31): one per subject, `mesh.assignment..`, reading the last one of each // first so it publishes only what changed, and no stream it defines. diff --git a/go/discover.go b/go/discover.go new file mode 100644 index 0000000..c7e5f90 --- /dev/null +++ b/go/discover.go @@ -0,0 +1,48 @@ +package stdio + +import ( + "encoding/json" + "fmt" + "time" +) + +// **Who answers a verb, asked by the bus's discovery** (D-root and D3 of the seat mesh-self-check; issue 1955): the +// runtime asks the services protocol on the account it runs as (ADR 0197) and answers each machine and module that +// says it serves the verb. The bundle never asks the bus itself. + +// Answerer is one module on one machine that answers a verb. +type Answerer struct { + Machine string `json:"machine"` + Module string `json:"module"` +} + +const discoverTimeout = 10 * time.Second + +// Discover is every module that answers verb (`.` or `.`), on node when one is named: runtime +// verb `mesh/discover {verb, node?}`, answered as a list. None answering is an empty list and no error; an error is a +// question that could not be asked. +func Discover(verb, node string) ([]Answerer, error) { + if verb == "" { + return nil, fmt.Errorf("discover names the verb it asks about") + } + c, err := live() + if err != nil { + return nil, err + } + params := map[string]any{"verb": verb} + if node != "" { + params["node"] = node + } + raw, err := c.ask("mesh/discover", params, discoverTimeout) + if err != nil { + return nil, err + } + out := []Answerer{} + if len(raw) == 0 || string(raw) == "null" { + return out, nil + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("the runtime's answer to discover is not readable: %w", err) + } + return out, nil +} diff --git a/go/log_test.go b/go/log_test.go index 98bba28..a0f7bbc 100644 --- a/go/log_test.go +++ b/go/log_test.go @@ -154,6 +154,17 @@ func logging(t *testing.T, r *logRuntime, run func() (any, error)) map[string]an if json.Unmarshal(lines.Bytes(), &m) != nil { continue } + if m.Method == "mesh/discover" { + // Who answers: one module on ace, none on any other machine named. + var p struct{ Verb, Node string } + _ = json.Unmarshal(m.Params, &p) + var result any = []map[string]string{{"machine": "ace", "module": "node-tools"}} + if p.Node != "" && p.Node != "ace" { + result = nil + } + send(map[string]any{"id": m.ID, "result": result}) + continue + } if strings.HasPrefix(m.Method, "mesh/log.") { go func(m message) { a := r.answer(m) @@ -214,6 +225,7 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { narrowed, since *LogPage newest []LogEntry watched *LogEntry + found, nobody []Answerer ) got := logging(t, r, func() (any, error) { changes := Log("changes") @@ -272,6 +284,16 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { watched = &e case <-time.After(5 * time.Second): } + // Who answers a verb, through the runtime (D-root, D3): one, then nobody on a machine none answers on. + if found, err = Discover("node-engine.root-free", ""); err != nil { + return nil, err + } + if nobody, err = Discover("node-engine.root-free", "g14"); err != nil { + return nil, err + } + if _, err := Discover("", ""); err == nil { + return nil, fmt.Errorf("a discover naming no verb was asked") + } return "done", nil }) if out, isError := text(t, got); isError || out != `"done"` { @@ -342,6 +364,9 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { p["from"] != "since" || p["since"] != "2026-10-11T18:00:00Z" || watched == nil || watched.Sequence != 6 { t.Fatalf("a watch asked %s %v and heard %+v", method, p, watched) } + if len(found) != 1 || found[0] != (Answerer{Machine: "ace", Module: "node-tools"}) || nobody == nil || len(nobody) != 0 { + t.Fatalf("discover found %+v, and on g14 %+v", found, nobody) + } if info == nil || info.Entries != 6 || info.MaxBytes != 1<<30 || info.First != 1 || info.Last != 6 || info.Bytes == 0 { t.Fatalf("the log's info is %+v", info) } From 8fe34677536e7338069cef2c1eb2de4d85c321ee Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:02:07 +0100 Subject: [PATCH 50/52] Let a provider not ready yet wait and keep serving what it served, so a restarted gitea no longer stops serving its consumers until its keys load (issue 1954) --- go/provider/provider.go | 22 +++++++++++++++++++++- go/provider/provider_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/go/provider/provider.go b/go/provider/provider.go index c183514..df315a5 100644 --- a/go/provider/provider.go +++ b/go/provider/provider.go @@ -215,6 +215,11 @@ func New(opts Options) *Provider { plans: map[string][]byte{}, provisions: map[string][]byte{}, said: map[string]string{}, poke: make(chan struct{}, 1)} } +// ErrNotReady is what a Make returns, wrapped or alone, when the provider cannot make a consumer's provision yet but +// will without anyone's hand (its keys not read since a restart): the entry waits and keeps serving what it served. +// Any other error is a failure, and its entry serves nothing. +var ErrNotReady = errors.New("not ready yet") + // What a pass did for one bind. const ( Answered = "answered" @@ -493,7 +498,22 @@ func (p *Provider) answer(ctx context.Context, e Entry, have []byte) Outcome { } } if p.opts.Make != nil { - if err := p.opts.Make(ctx, Consumer{Entry: e, Credential: credential}); err != nil { + if err := p.opts.Make(ctx, Consumer{Entry: e, Credential: credential}); errors.Is(err, ErrNotReady) { + // **Not ready is no failure** (the regression of 2026-10-30: gitea after a restart, its keys not read yet, + // stopped serving every consumer): the entry says waits with why, and keeps what the last entry served, so a + // consumer goes on with it until the provider can make it again. + why := "the provider is not ready to make it yet: " + redact(err.Error(), credential) + waiting := e + var prev Entry + if len(have) > 0 && json.Unmarshal(have, &prev) == nil && prev.State == provisions.Provided { + waiting.Values, waiting.VaultName, waiting.User = prev.Values, prev.VaultName, prev.User + } else { + waiting.Values, waiting.VaultName, waiting.User = nil, "", "" + } + waiting.State, waiting.Why = provisions.Waits, why + p.write(ctx, waiting, have, Outcome{Key: key, Status: Waiting, Why: why}) + return Outcome{Key: key, Status: Waiting, Why: why} + } else if err != nil { why := "the provider could not make it: " + redact(err.Error(), credential) // Said in its entry, which the holder of mesh-provisioner reads (ADR 0570 clause 3), with nothing served. failed := e diff --git a/go/provider/provider_test.go b/go/provider/provider_test.go index 806bb34..eec9d1b 100644 --- a/go/provider/provider_test.go +++ b/go/provider/provider_test.go @@ -525,6 +525,33 @@ func TestAFailedMakeIsSaidWithoutTheCredential(t *testing.T) { strings.Contains(string(raw), "hunter2") { t.Fatalf("the failed entry is %s", raw) } + + // A provider not ready yet (gitea after a restart, its keys not read; the regression of 2026-10-30) is no failure: + // a first entry waits serving nothing, and once made, a later not ready keeps serving what it served. + m.fail = fmt.Errorf("keys not read yet: %w", ErrNotReady) + _ = bus.Delete(context.Background(), BucketOf("postgres"), "ace.letta.postgres-database") + if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Waiting { + t.Fatalf("not ready at first: %+v", out) + } + raw = bus.get(BucketOf("postgres"), "ace.letta.postgres-database") + if err := json.Unmarshal(raw, &e); err != nil || e.State != "waits" || !strings.Contains(e.Why, "keys not read yet") || e.Values != nil { + t.Fatalf("the first entry not ready is %s", raw) + } + m.fail = nil + if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Answered { + t.Fatalf("made: %+v", out) + } + var made Entry + _ = json.Unmarshal(bus.get(BucketOf("postgres"), "ace.letta.postgres-database"), &made) + m.fail = ErrNotReady + if out, _ := p.Pass(context.Background()); len(out) != 1 || out[0].Status != Waiting { + t.Fatalf("not ready after a restart: %+v", out) + } + raw = bus.get(BucketOf("postgres"), "ace.letta.postgres-database") + if err := json.Unmarshal(raw, &e); err != nil || e.State != "waits" || e.VaultName != made.VaultName || e.User != made.User || + fmt.Sprint(e.Values) != fmt.Sprint(made.Values) || len(made.Values) == 0 { + t.Fatalf("not ready after it served, the entry is %s, it served %+v", raw, made) + } } // Run watches the binds on the stand-in bus: a consumer that binds is answered, and one that unbinds is removed, From 007b6fd45fc7e7bfa93e76f6b9009725f881e29f Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:02:07 +0100 Subject: [PATCH 51/52] Drop stdio.Discover, the self-check's discovery and its read of the withheld hearings, which no probe uses now that the operator dropped D-root, D3 and D-heard (issue 1952) --- go/buscompose/buscompose_test.go | 18 ++++-------- go/buscompose/health_grants.go | 3 -- go/buscompose/nats.go | 5 ---- go/buscompose/userlist_switch.go | 3 +- go/discover.go | 48 -------------------------------- go/log_test.go | 25 ----------------- 6 files changed, 8 insertions(+), 94 deletions(-) delete mode 100644 go/discover.go diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index bd56f70..5e76548 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -41,12 +41,6 @@ var createdAdvisory = regexp.MustCompile(`"\$JS\.EVENT\.ADVISORY\.STREAM\.CREATE // write. func beyondTheControllers(user string) map[string]bool { out := map[string]bool{} - // The self-check's runtime asks who answers (D-root, D3). - if strings.HasSuffix(user, "."+SelfCheckModule) { - for _, s := range discovering() { - out[s] = true - } - } if strings.HasSuffix(user, "."+WatchmanModule) { for _, s := range append(WatchmanDeadLetterGrants(nil), WatchmanHears()...) { out[s] = true @@ -620,7 +614,7 @@ func healthChecks(t *testing.T) { users, err := Users(Records{Nodes: []string{"a"}, Assigned: map[string][]Declared{"a": { {Module: "mesh-conditions", State: []Bucket{bucket("mesh-conditions", "conditions"), bucket("mesh-conditions", "takeover")}, Reads: []string{"mesh-self-check.findings", "mesh-healers.attempts", ControllerConditions}}, - {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good", GrantsWithheldRead}}, + {Module: "mesh-self-check", State: []Bucket{bucket("mesh-self-check", "findings")}, Reads: []string{"restic.good"}}, // Issue 1955 L7: the grants part keeps the hearings it withholds; the self-check reads them. {Module: GrantsModule, State: []Bucket{bucket(GrantsModule, GrantsWithheldState)}}, {Module: "mesh-healers", State: []Bucket{bucket("mesh-healers", "findings"), bucket("mesh-healers", "attempts"), bucket("mesh-healers", "would")}}, @@ -665,9 +659,8 @@ func healthChecks(t *testing.T) { "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.c1.$KV.mesh-provisioner_plans.a.keycloak"}, // ADR 0571 clause 3: restic's bundle writes each item's last good night as its own state; the self-check reads it. - "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data", "$SRV.INFO.node-tools", "$SRV.PING", - "$JS.API.DIRECT.GET.KV_mesh-grants_withheld.$KV.mesh-grants_withheld." + WithheldKey(Withheld{Node: "a", Module: "reader", Stream: "MAIL"})}, - "a.mesh-grants": {"$KV.mesh-grants_withheld.reader.mail.a"}, + "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"}, + "a.mesh-grants": {"$KV.mesh-grants_withheld.reader.mail.a"}, "a.restic": {"$JS.API.DIRECT.GET.KV_disk-load_readings.$KV.disk-load_readings.a.pg.data", "$KV.restic_good.a.pg.data", // ADR 0571 clause 7: its own membership from the grants part's state. "$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.a.restic"}, @@ -687,14 +680,15 @@ func healthChecks(t *testing.T) { // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). "a.mesh-conditions": {"$KV.mesh-self-check_findings.x", "$KV.mesh-healers_attempts.x", "$KV.mesh-controller_conditions.x"}, "a.mesh-self-check": {"$KV.mesh-conditions_conditions.x", "$KV.mesh-conditions_takeover.switch", "$KV.restic_good.a.pg.data", - "$KV.mesh-grants_withheld.reader.mail.a"}, + "$KV.mesh-grants_withheld.reader.mail.a", "$SRV.INFO", "$SRV.PING.node-tools", + "$JS.API.DIRECT.GET.KV_mesh-grants_withheld.$KV.mesh-grants_withheld.reader.mail.a"}, // Only mesh-provisioner writes provisions and plans, and it writes none of the grants part's states (ADR 0570 // clauses 1, 8). "a.mesh-provisioner": {"$KV.mesh-grants_accounts.a.m", "$KV.mesh-grants_user-list.sealed"}, "a.mesh-vault": {"$KV.mesh-provisioner_provisions.a.keycloak.postgres-database", "$KV.mesh-provisioner_plans.a.keycloak", "mesh.seat.mesh-artifact-sweep.event.artifacts-gone"}, // Each writes its own state alone, and these users are composed. - "a.disk-load": {"$KV.restic_good.a.pg.data", "$SRV.INFO", "$SRV.PING.node-tools"}, + "a.disk-load": {"$KV.restic_good.a.pg.data"}, "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, // Another machine's plans are not its to read; no client writes or reads the needs any more. "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.b.keycloak", diff --git a/go/buscompose/health_grants.go b/go/buscompose/health_grants.go index 916853d..e31a935 100644 --- a/go/buscompose/health_grants.go +++ b/go/buscompose/health_grants.go @@ -170,6 +170,3 @@ func keepsDeadLetters(p Principal, pattern string) bool { return isController(p, pattern) || (p.Kind == KindModule && p.Module == WatchmanModule && pattern == deadLetterPrefix+">") } - -// SelfCheckModule is the module holding the seat mesh-self-check, whose runtime asks the bus who answers (D-root, D3). -const SelfCheckModule = health.SelfCheckModule diff --git a/go/buscompose/nats.go b/go/buscompose/nats.go index 64b6adf..61bdfa9 100644 --- a/go/buscompose/nats.go +++ b/go/buscompose/nats.go @@ -1014,11 +1014,6 @@ func PermissionsFor(p Principal) (Permissions, error) { // And the words its watchdogs hear (ADR 0564 clause 4, Limits 3, 11). sub = append(sub, WatchmanHears()...) } - // The self-check asks who answers a verb (D-root, D3): its runtime, on the self-check's own account (runs-as), - // asks the bus's discovery, its replies to its own inbox; no other module's account may. - if p.Module == SelfCheckModule { - pub = append(pub, discovering()...) - } // The grants part alone publishes the modules' memberships into the assignments stream (novox/hq ADR 0547 §5; // the operator, 2026-10-25 22:31): one per subject, `mesh.assignment..`, reading the last one of each // first so it publishes only what changed, and no stream it defines. diff --git a/go/buscompose/userlist_switch.go b/go/buscompose/userlist_switch.go index dc187d8..9f57881 100644 --- a/go/buscompose/userlist_switch.go +++ b/go/buscompose/userlist_switch.go @@ -130,7 +130,8 @@ func withNamedReaders(d Declared) Declared { // **The hearings the grants part withholds, on the bus** (issue 1955 L7, mesh-delivery e564eaf): a hearing of a restricted // stream a module asks in the tool runner, withheld because only an account of its own may hear it (ADR 0376 §2), kept // in the grants part's state `withheld`, one key `..` (the controller's condition id -// of it), written on change only and deleted once heard. The self-check's probe D-heard reads it there. +// of it), written on change only and deleted once heard; the grants part's tools read it (D-heard was dropped on +// 2026-10-30, so no other module reads it). const ( GrantsWithheldState = "withheld" GrantsWithheldBucket = GrantsModule + "_" + GrantsWithheldState diff --git a/go/discover.go b/go/discover.go deleted file mode 100644 index c7e5f90..0000000 --- a/go/discover.go +++ /dev/null @@ -1,48 +0,0 @@ -package stdio - -import ( - "encoding/json" - "fmt" - "time" -) - -// **Who answers a verb, asked by the bus's discovery** (D-root and D3 of the seat mesh-self-check; issue 1955): the -// runtime asks the services protocol on the account it runs as (ADR 0197) and answers each machine and module that -// says it serves the verb. The bundle never asks the bus itself. - -// Answerer is one module on one machine that answers a verb. -type Answerer struct { - Machine string `json:"machine"` - Module string `json:"module"` -} - -const discoverTimeout = 10 * time.Second - -// Discover is every module that answers verb (`.` or `.`), on node when one is named: runtime -// verb `mesh/discover {verb, node?}`, answered as a list. None answering is an empty list and no error; an error is a -// question that could not be asked. -func Discover(verb, node string) ([]Answerer, error) { - if verb == "" { - return nil, fmt.Errorf("discover names the verb it asks about") - } - c, err := live() - if err != nil { - return nil, err - } - params := map[string]any{"verb": verb} - if node != "" { - params["node"] = node - } - raw, err := c.ask("mesh/discover", params, discoverTimeout) - if err != nil { - return nil, err - } - out := []Answerer{} - if len(raw) == 0 || string(raw) == "null" { - return out, nil - } - if err := json.Unmarshal(raw, &out); err != nil { - return nil, fmt.Errorf("the runtime's answer to discover is not readable: %w", err) - } - return out, nil -} diff --git a/go/log_test.go b/go/log_test.go index a0f7bbc..98bba28 100644 --- a/go/log_test.go +++ b/go/log_test.go @@ -154,17 +154,6 @@ func logging(t *testing.T, r *logRuntime, run func() (any, error)) map[string]an if json.Unmarshal(lines.Bytes(), &m) != nil { continue } - if m.Method == "mesh/discover" { - // Who answers: one module on ace, none on any other machine named. - var p struct{ Verb, Node string } - _ = json.Unmarshal(m.Params, &p) - var result any = []map[string]string{{"machine": "ace", "module": "node-tools"}} - if p.Node != "" && p.Node != "ace" { - result = nil - } - send(map[string]any{"id": m.ID, "result": result}) - continue - } if strings.HasPrefix(m.Method, "mesh/log.") { go func(m message) { a := r.answer(m) @@ -225,7 +214,6 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { narrowed, since *LogPage newest []LogEntry watched *LogEntry - found, nobody []Answerer ) got := logging(t, r, func() (any, error) { changes := Log("changes") @@ -284,16 +272,6 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { watched = &e case <-time.After(5 * time.Second): } - // Who answers a verb, through the runtime (D-root, D3): one, then nobody on a machine none answers on. - if found, err = Discover("node-engine.root-free", ""); err != nil { - return nil, err - } - if nobody, err = Discover("node-engine.root-free", "g14"); err != nil { - return nil, err - } - if _, err := Discover("", ""); err == nil { - return nil, fmt.Errorf("a discover naming no verb was asked") - } return "done", nil }) if out, isError := text(t, got); isError || out != `"done"` { @@ -364,9 +342,6 @@ func TestABundleKeepsALogThroughTheRuntime(t *testing.T) { p["from"] != "since" || p["since"] != "2026-10-11T18:00:00Z" || watched == nil || watched.Sequence != 6 { t.Fatalf("a watch asked %s %v and heard %+v", method, p, watched) } - if len(found) != 1 || found[0] != (Answerer{Machine: "ace", Module: "node-tools"}) || nobody == nil || len(nobody) != 0 { - t.Fatalf("discover found %+v, and on g14 %+v", found, nobody) - } if info == nil || info.Entries != 6 || info.MaxBytes != 1<<30 || info.First != 1 || info.Last != 6 || info.Bytes == 0 { t.Fatalf("the log's info is %+v", info) } From 6e4a3014cbde924dcd24158369cbe9ad287ce2c9 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:14:53 +0100 Subject: [PATCH 52/52] Keep the provisioner client's needs grant for one release, removed after the lease write, so the client before V3 still writes its needs while its delivery reaches each node (issue 1954) --- go/buscompose/buscompose_test.go | 10 ++++++---- go/buscompose/provisioner.go | 12 ++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go/buscompose/buscompose_test.go b/go/buscompose/buscompose_test.go index 5e76548..722171e 100644 --- a/go/buscompose/buscompose_test.go +++ b/go/buscompose/buscompose_test.go @@ -657,7 +657,9 @@ func healthChecks(t *testing.T) { "a.mesh-hand-acts": {"$KV.mesh-hand-acts_warrants.mesh-config.ask-1", "mesh.log.mesh-hand-acts.acts.act-1"}, "a.disk-load": {"$KV.disk-load_readings.a.pg.data"}, "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.a.keycloak", - "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.c1.$KV.mesh-provisioner_plans.a.keycloak"}, + "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.c1.$KV.mesh-provisioner_plans.a.keycloak", + // The needs, for the client before V3, until the lease write (review round 3). + "$KV." + ProvisionerNeedsBucket + ".a.keycloak.db.x"}, // ADR 0571 clause 3: restic's bundle writes each item's last good night as its own state; the self-check reads it. "a.mesh-self-check": {"$JS.API.DIRECT.GET.KV_restic_good.$KV.restic_good.a.pg.data"}, "a.mesh-grants": {"$KV.mesh-grants_withheld.reader.mail.a"}, @@ -690,10 +692,10 @@ func healthChecks(t *testing.T) { // Each writes its own state alone, and these users are composed. "a.disk-load": {"$KV.restic_good.a.pg.data"}, "a.mesh-delivery-builder": {"$KV.disk-load_readings.a.pg.data"}, - // Another machine's plans are not its to read; no client writes or reads the needs any more. + // Another machine's plans are not its to read, nor another machine's needs, which it keeps for the client before + // V3 until the lease write. "provisioner.a": {"$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.$KV.mesh-provisioner_plans.b.keycloak", - "$KV.mesh-provisioner_plans.a.keycloak", "$KV." + ProvisionerNeedsBucket + ".a.keycloak.db.x", - "$JS.API.DIRECT.GET.KV_" + ProvisionerNeedsBucket + ".$KV." + ProvisionerNeedsBucket + ".a.keycloak.db.x"}, + "$KV.mesh-provisioner_plans.a.keycloak", "$KV." + ProvisionerNeedsBucket + ".b.keycloak.db.x"}, // Only the assigner writes seat-holders. "a.artifact-store-tools": {"$KV.mesh-delivery-assigner_seat-holders.mesh-broker"}, // Only disk-load writes its readings. diff --git a/go/buscompose/provisioner.go b/go/buscompose/provisioner.go index 19cd361..5221bf7 100644 --- a/go/buscompose/provisioner.go +++ b/go/buscompose/provisioner.go @@ -71,10 +71,14 @@ var ( ) // ProvisionerWrites are the server's buckets a client writes, and reads, its own machine's keys of (`.>`) -// and no other's: what it applied, and the items of the collections on its machine. A client cannot say another -// machine applied a file. It records no needs: since the client reads its machine's plans from the seat -// mesh-provisioner's `plans` (novox/hq ADR 0570 clause 2; the client's V3), the needs bucket is granted to none. -var ProvisionerWrites = []string{ProvisionerAppliedBucket, ProvisionerCollectBucket} +// and no other's: the needs it records, what it applied, and the items of the collections on its machine. A client +// cannot write another machine's needs, or say another machine applied a file. +// +// **The needs stay granted for one release, and are removed after the lease write** (review round 3 of design 47's +// release): the client's V3 reads its machine's plans from the seat mesh-provisioner's `plans` (novox/hq ADR 0570 clause +// 2) and records no needs, but the client before it still runs on a node until its delivery reaches it, and it writes +// them. +var ProvisionerWrites = []string{ProvisionerNeedsBucket, ProvisionerAppliedBucket, ProvisionerCollectBucket} // VerbsTheProvisionerAsksTheVault are the vault's verbs a provisioner client asks in its own name (ADR 0349 // clause 11): `get`, a secret its machine's declaration names, answered sealed to the asker's key. On the subject