diff --git a/checks/credentialform/go.mod b/checks/credentialform/go.mod new file mode 100644 index 00000000..1793672d --- /dev/null +++ b/checks/credentialform/go.mod @@ -0,0 +1,5 @@ +module checks/credentialform + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/checks/credentialform/go.sum b/checks/credentialform/go.sum new file mode 100644 index 00000000..f8bfbd2b --- /dev/null +++ b/checks/credentialform/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/checks/credentialform/main.go b/checks/credentialform/main.go new file mode 100644 index 00000000..e5a15729 --- /dev/null +++ b/checks/credentialform/main.go @@ -0,0 +1,70 @@ +// Package credentialform refuses an offer that does not say the form of its credential: an own secret, an issuer, or +// `form` pair or none (the lead, 2026-10-29). It reads every modules/*/module.json through mesh-sdk's +// CredentialFormProblems. +package credentialform + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Problems is every offer of the manifests under root/modules that says no credential form, one line each. +func Problems(root string) ([]string, error) { + files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(files) + var out []string + reads := map[string]bool{} + var binders []string + for _, f := range files { + raw, err := os.ReadFile(f) + if err != nil { + return nil, err + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", f, err) + } + out = append(out, catalogue.CredentialFormProblems(m)...) + var kept struct { + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` + } + _ = json.Unmarshal(raw, &kept) // reads may hold objects in another module; only the provisioner's are read + if m.Module == "mesh-provisioner" { + for _, r := range kept.Reads { + reads[r] = true + } + } + if len(m.Provides) > 0 && keepsBound(kept.State) { + binders = append(binders, m.Module) + } + } + for _, module := range binders { + if !reads[module+".bound"] { + out = append(out, fmt.Sprintf("%s provides and keeps a `bound` state, and mesh-provisioner's manifest does not read %s.bound", module, module)) + } + } + return out, nil +} + +// keepsBound says whether a manifest's `state` names `bound`, as a plain name or an object with a name. +func keepsBound(state []json.RawMessage) bool { + for _, s := range state { + var name string + var obj struct { + Name string `json:"name"` + } + if json.Unmarshal(s, &name) == nil && name == "bound" || json.Unmarshal(s, &obj) == nil && obj.Name == "bound" { + return true + } + } + return false +} diff --git a/checks/credentialform/main_test.go b/checks/credentialform/main_test.go new file mode 100644 index 00000000..33c1c4f4 --- /dev/null +++ b/checks/credentialform/main_test.go @@ -0,0 +1,49 @@ +package credentialform + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func aCatalogue(t *testing.T, manifests map[string]string) string { + t.Helper() + root := t.TempDir() + for module, body := range manifests { + dir := filepath.Join(root, "modules", module) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +// An offer that says no form is refused and named; one that says pair, none or an own secret passes; and the real +// catalogue is clean. +func TestEveryOfferSaysTheFormOfItsCredential(t *testing.T) { + root := aCatalogue(t, map[string]string{ + "silent": `{"module":"silent","provides":[{"name":"thing","scope":"mesh"}]}`, + "binder": `{"module":"binder","provides":[{"name":"d","credential":{"form":"pair"}}],"state":["bound"]}`, + "mesh-provisioner": `{"module":"mesh-provisioner","reads":["other.bound"]}`, + "says": `{"module":"says","provides":[{"name":"a","credential":{"form":"pair"}},{"name":"b","credential":{"form":"none"}},{"name":"c","credential":{"own":"jwt"}}]}`, + }) + got, err := Problems(root) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !strings.Contains(got[0], `silent provides "thing"`) || + !strings.Contains(got[1], "binder") || !strings.Contains(got[1], "mesh-provisioner") { + t.Fatalf("refused %v, wanted the silent offer and the binder the provisioner does not read", got) + } + real, err := Problems(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + if len(real) != 0 { + t.Fatalf("the catalogue has offers with no credential form:\n%s", strings.Join(real, "\n")) + } +} diff --git a/checks/credentialform/reads-outside b/checks/credentialform/reads-outside new file mode 100644 index 00000000..8ff16290 --- /dev/null +++ b/checks/credentialform/reads-outside @@ -0,0 +1,3 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what each offer says of its credential's form. +modules/*/module.json diff --git a/checks/handacts/fallbacks b/checks/handacts/fallbacks new file mode 100644 index 00000000..c4fdf709 --- /dev/null +++ b/checks/handacts/fallbacks @@ -0,0 +1,8 @@ +# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each ` `: +# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until +# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any +# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write +# (novox/hq ADR 0565 §11; issue 1953, issue 774). +mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first +mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first +mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first diff --git a/checks/handacts/go.mod b/checks/handacts/go.mod new file mode 100644 index 00000000..29e4c52b --- /dev/null +++ b/checks/handacts/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/handacts + +go 1.22 diff --git a/checks/handacts/main.go b/checks/handacts/main.go new file mode 100644 index 00000000..1a222d60 --- /dev/null +++ b/checks/handacts/main.go @@ -0,0 +1,109 @@ +// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every +// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes +// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the +// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and +// an entry whose module no longer invokes the verb is refused too. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks. +var Verbs = []string{"hand-act", "warranted", "drill"} + +const ( + invokePrefix = "seat:mesh-controller." + Allowed = "checks/handacts/fallbacks" +) + +// Findings is every refusal in the catalog at root, sorted. +func Findings(root string) ([]string, error) { + allowed := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(root, Allowed)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]+" "+f[1]] = true + } else if len(f) > 0 && !strings.HasPrefix(f[0], "#") { + return nil, fmt.Errorf("%s: %q is not ` `", Allowed, line) + } + } + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var out []string + invoked := map[string]bool{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Invokes []string `json:"invokes"` + } + if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" { + continue + } + for _, inv := range m.Invokes { + verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix) + if !ok || !contains(Verbs, verb) { + continue + } + invoked[m.Module+" "+verb] = true + if !allowed[m.Module+" "+verb] { + out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+ + "or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed)) + } + } + } + for entry := range allowed { + if !invoked[entry] { + out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks", + Allowed, entry)) + } + } + sort.Strings(out) + return out, nil +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: handacts ") + os.Exit(2) + } + found, err := Findings(os.Args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } + fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named") +} diff --git a/checks/handacts/main_test.go b/checks/handacts/main_test.go new file mode 100644 index 00000000..61e197e6 --- /dev/null +++ b/checks/handacts/main_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is +// refused (novox/hq ADR 0565 §11's catalog test). +func TestFindings(t *testing.T) { + for _, row := range []struct { + name string + files map[string]string + want string // empty: clean + }{ + {"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""}, + {"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`}, + "a invokes seat:mesh-controller.warranted"}, + {"a named fallback, and the controller's own", map[string]string{ + "modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`, + "modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`, + Allowed: "# why\na drill fallback until the lease write\n"}, ""}, + {"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`}, + {"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""}, + } { + t.Run(row.name, func(t *testing.T) { + root := t.TempDir() + for path, text := range row.files { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := Findings(root) + if err != nil { + t.Fatal(err) + } + if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) { + t.Fatalf("findings = %q, want one containing %q", all, row.want) + } + }) + } +} + +// The run of this check over the real tree. +func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) { + got, err := Findings(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + for _, f := range got { + t.Error(f) + } +} diff --git a/checks/handacts/reads-outside b/checks/handacts/reads-outside new file mode 100644 index 00000000..61f9814a --- /dev/null +++ b/checks/handacts/reads-outside @@ -0,0 +1,3 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what it invokes. +modules/*/module.json diff --git a/checks/health/go.mod b/checks/health/go.mod index a4d884a2..5ec7f46c 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/checks/health/go.sum b/checks/health/go.sum index fc336ec2..f8bfbd2b 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/checks/noplacement/assignments-readers b/checks/noplacement/assignments-readers new file mode 100644 index 00000000..64985d46 --- /dev/null +++ b/checks/noplacement/assignments-readers @@ -0,0 +1,4 @@ +# The modules of this catalogue that still read the controller's assignments (mesh-controller.assignments), each with +# why. The check of this directory (noplacement) refuses any other module that reads them, and an entry here whose module no longer +# does: the list only shrinks, and it is empty before the controller's lease is written to a holder that is no +# controller (novox/hq ADR 0549 §3, §6; issue 774). diff --git a/checks/noplacement/go.mod b/checks/noplacement/go.mod index a09ac5ec..782a236b 100644 --- a/checks/noplacement/go.mod +++ b/checks/noplacement/go.mod @@ -1,3 +1,7 @@ module git.novox.be/novox/mesh-catalog/checks/noplacement go 1.22 + +require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0 + +replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk diff --git a/checks/noplacement/main.go b/checks/noplacement/main.go index 3de240c4..cd145e32 100644 --- a/checks/noplacement/main.go +++ b/checks/noplacement/main.go @@ -8,7 +8,13 @@ // 2. a `particulars` file at the root (the mesh's node names come from the checker's run, §16); // // 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…` -// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`. +// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`; +// +// 4. a manifest, other than mesh-controller's, that reads the controller's assignments (`mesh-controller.assignments`): +// every reader takes placement from the seat mesh-placement (novox/hq ADR 0549 §3, ADR 0567), and the lease write is +// refused while one still reads them (ADR 0549 §6; issue 774). The modules that still read them for what the +// placement does not carry are named in checks/noplacement/assignments-readers, each with why; that list only +// shrinks: an entry whose module no longer reads them is refused too. // // go run . // @@ -55,9 +61,74 @@ func Findings(root string) ([]string, error) { out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s)) } } + readers, err := assignmentsReaders(root) + if err != nil { + return nil, err + } + out = append(out, readers...) + sort.Strings(out) return out, nil } +// AssignmentsRead is the controller's assignments as a manifest reads them, and Allowed the file, relative to the +// catalogue's root, that names the modules still reading them: ` ` per line, `#` a comment. +const ( + AssignmentsRead = "mesh-controller.assignments" + Allowed = "checks/noplacement/assignments-readers" +) + +// assignmentsReaders is every finding on the reads of the controller's assignments: a module other than the controller +// reading them that Allowed does not name, and a module Allowed names that no longer reads them (issue 774). +func assignmentsReaders(root string) ([]string, error) { + allowed := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(root, Allowed)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]] = true + } + } + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var found []string + reads := map[string]bool{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Reads []string `json:"reads"` + } + if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" { + continue // a manifest that does not read is the manifest checks' to refuse + } + for _, r := range m.Reads { + if strings.TrimSpace(r) != AssignmentsRead { + continue + } + reads[m.Module] = true + if !allowed[m.Module] { + found = append(found, fmt.Sprintf("%s reads %s: read where modules run from mesh-delivery-assigner.placement, "+ + "the seat mesh-placement's (novox/hq ADR 0549 §3, ADR 0567); the lease write is refused while a module reads "+ + "the assignments (ADR 0549 §6)", m.Module, AssignmentsRead)) + } + } + } + for module := range allowed { + if !reads[module] { + found = append(found, fmt.Sprintf("%s names %s, which no longer reads %s: take it off the list, which only shrinks", + Allowed, module, AssignmentsRead)) + } + } + return found, nil +} + // named collects the strings of a decoded manifest that start with one of the Prefixes. func named(v any, into []string) []string { switch v := v.(type) { @@ -95,5 +166,5 @@ func main() { if len(found) > 0 { os.Exit(1) } - fmt.Println("no placement directory, and no manifest reads the indexer's placement") + fmt.Println("no placement directory, no manifest reads the indexer's placement, and only the modules listed read the controller's assignments") } diff --git a/checks/noplacement/main_test.go b/checks/noplacement/main_test.go index 0ffc320f..efa884dc 100644 --- a/checks/noplacement/main_test.go +++ b/checks/noplacement/main_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-catalog/checks/trunk" ) func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) { @@ -13,12 +15,18 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) { files map[string]string want string // empty: clean }{ - {"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-controller.assignments"]}`}, ""}, + {"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.placement"]}`}, ""}, {"a placement directory", map[string]string{"placement/ace.json": `{}`}, "placement: the repository holds no placement"}, {"a particulars file", map[string]string{"particulars": "ace\n"}, "particulars: the repository lists no node names"}, {"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`}, {"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"}, {"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"}, + // Issue 774 (ADR 0549 §3, §6): a reader of the controller's assignments is refused unless the list names it, the + // controller's own manifest is not judged, and a name on the list whose module no longer reads them is refused. + {"an unlisted reader of the assignments", map[string]string{"modules/r/module.json": `{"module":"r","reads":["mesh-controller.assignments"]}`}, "r reads mesh-controller.assignments"}, + {"a listed reader and the controller", map[string]string{"modules/l/module.json": `{"module":"l","reads":["mesh-controller.assignments"]}`, + "modules/mesh-controller/module.json": `{"module":"mesh-controller","reads":["mesh-controller.assignments"]}`, Allowed: "# why\nl its plan\n"}, ""}, + {"a listed module that reads them no more", map[string]string{Allowed: "gone no longer reads\n"}, "names gone, which no longer reads"}, {"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""}, } { t.Run(row.name, func(t *testing.T) { @@ -52,4 +60,29 @@ func TestTheCatalogueHoldsNoPlacementOfTheRepositoryPath(t *testing.T) { for _, f := range got { t.Error(f) } + // Issue 774 (ADR 0549 §3, §6; the comparison of novox/hq ADR 0529 clause 2): the list of modules still reading the + // controller's assignments gains none the trunk's copy at the merge base does not hold. Its first copy gains nothing. + path, ok := trunk.Copy(t, Allowed) + if !ok { + return + } + names := func(path string) map[string]bool { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + out := map[string]bool{} + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") { + out[f[0]] = true + } + } + return out + } + base := names(path) + for module := range names(filepath.Join("..", "..", Allowed)) { + if !base[module] { + t.Errorf("%s gains %s, which the trunk's copy does not hold: the list only shrinks", Allowed, module) + } + } } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go new file mode 100644 index 00000000..1f010869 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go @@ -0,0 +1,129 @@ +package main + +// The artifact store's door and the bus, as the holder reaches them through this module's runtime. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "artifactstoretools/internal/registry" +) + +// door is registry.Store at the store's address as this machine reaches it. +type door struct{ s registry.Store } + +func (d *door) Address() string { return d.s.Address } +func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) } +func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) } +func (d *door) IsIndex(ctx context.Context, r string) (bool, error) { + return d.s.IsIndex(ctx, r) +} +func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) } +func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error { + spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching) + if err == nil { + d.s.Spare = spare + } + return err +} + +// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's +// reach, each time, so a store that moved is reached where it is now. +func DoorAt(file string) func() (Door, error) { + return func() (Door, error) { + if file == "" { + return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address") + } + raw, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("the artifact store's address: %w", err) + } + address := strings.TrimSpace(string(raw)) + if address == "" { + return nil, fmt.Errorf("the artifact store's address is empty in %s", file) + } + return &door{s: registry.Store{Address: address}}, nil + } +} + +// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event. +type runtimeBus struct{} + +func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +func (runtimeBus) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} +func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) { + return stdio.State(state).History(key) +} +func (runtimeBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + page, err := stdio.Log(read).Since(at, "", 0) + for err == nil { + out = append(out, page.Entries...) + if page.Next == 0 { + return out, nil + } + page, err = stdio.Log(read).Read(page.Next, "", 0) + } + return nil, err +} + +func (runtimeBus) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} +func (runtimeBus) Delete(state, key string) error { return stdio.State(state).Delete(key) } +func (runtimeBus) Publish(subject string, body any) error { + return stdio.SeatPublish(subject, body, "") +} + +// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and +// observes again; a want changed observes again. Event-driven: nothing reads them on a timer. +func follow(h *Holder) { + <-stdio.Served() + watch := func(state string, handle func(stdio.StateChange) error) { + for { + if err := stdio.State(state).Watch("", handle); err == nil { + return + } else { + h.logf("watching %s: %v; asked again in 10s", state, err) + } + time.Sleep(10 * time.Second) + } + } + observe := func() { + if err := h.Observe(context.Background()); err != nil { + h.logf("image-missing could not be observed: %v", err) + } + } + watch(BuildsRead, func(c stdio.StateChange) error { + if c.Current || c.Op != stdio.OpPut { + return nil + } + var b struct { + State string `json:"state"` + } + if json.Unmarshal(c.Value, &b) == nil && b.State == "done" { + go func() { h.AfterBuild(c.Key); observe() }() + } + return nil + }) + watch(WantsRead, func(c stdio.StateChange) error { + if !c.Current { + go observe() + } + return nil + }) + observe() +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go new file mode 100644 index 00000000..5325b52f --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -0,0 +1,1020 @@ +package main + +// **The seat mesh-artifact-sweep** (novox/hq ADR 0571 clause 2, issue 1955): this module holds the artifact store's sweep, out +// of the controller. Its keep set is the store's own listing joined with every want's fingerprint (the assigner's +// `wants`) and each module's five newest builds (the builder's done keys, each build's end read from the key's history; +// ADR 0526 §1): a digest a want names is kept however old; a build beyond the five newest that no want names is +// eligible; a manifest in the store that no build names is unrecorded and never removed (ADR 0189 §10, ADR 0251 §6). +// +// At each build's end (a done key written) it sweeps as the controller's take-in did: every kept archive held first +// (ADR 0251 §8), then what is eligible let go of, oldest first, within the same bound per sweep, an index left for a +// `collect` a person confirms (ADR 0251 §7), each deletion recorded in the hand-act log (seat mesh-hand-acts) and said +// as artifacts-gone. **Nothing is let go until the controller's lease names a holder that is no controller**: until +// then the controller's take-in sweeps, and two sweeps never run at once. It also observes S25, image-missing: a want's +// image the store does not hold, a finding in its own state `findings` the condition keeper reads. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" + + "artifactstoretools/internal/registry" +) + +// What the holder reads and keeps. +const ( + WantsRead = delivery.AssignerModule + ".wants" + BuildsRead = delivery.BuilderModule + ".builds" + HavesRead = "node-engine.have" + // ManifestsRead is the indexer's manifests: the images a manifest pins by itself. + ManifestsRead = delivery.IndexerModule + ".manifests" + LeaseRead = buscompose.ControllerLease + // FindingsState is the holder's own findings (health.FindingsState): image-missing. + FindingsState = health.FindingsState + // Module is this module, the observer its findings name. + Module = "artifact-store-tools" + // GoneSubject is where the seat says artifacts-gone. + GoneSubject = "mesh.seat." + catalogue.ArtifactSweepSeat + ".event." + delivery.ArtifactsGoneEvent +) + +// The sweep's bounds, the controller's (cmd/mesh-controller/collect.go mostPerSweep, sweepBudget). +const ( + KeptBuilds = 5 + MostPerSweep = 500 + MostAtAll = 5000 + SweepBudget = 45 * time.Second + ImageMissing = "image-missing" +) + +// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves, the lease), its own +// findings, and its seat's event. The runtime in main; a test's in memory. +type Bus interface { + Keys(state string) ([]string, error) + Get(state, key string) ([]byte, bool, error) + History(state, key string) ([]stdio.StateRevision, error) + // LogSince is every entry of a read log from a time on (stdio.Log(read).Since). + LogSince(read string, at time.Time) ([]stdio.LogEntry, error) + Put(state, key string, value any) error + Delete(state, key string) error + Publish(subject string, body any) error +} + +// Door is the artifact store's own door: registry.Store, behind an interface so a test needs no store. +type Door interface { + Address() string + Hold(ctx context.Context, reference string) (bool, error) + Holds(ctx context.Context, reference string) error + IsIndex(ctx context.Context, reference string) (bool, error) + LetGo(ctx context.Context, reference string) error + // SpareKept sets the platform manifests a kept index names aside before an index is let go of (ADR 0257). + SpareKept(ctx context.Context, keptImages, touching []string) error +} + +// Holder is the seat's holder. +type Holder struct { + Bus Bus + Door func() (Door, error) + // Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected. + Listed func(ctx context.Context) (*View, error) + Acts handacts.Recorder + Now func() time.Time + Logf func(string, ...any) + + sweep sync.Mutex // one sweep at a time + mu sync.Mutex + ended map[string]time.Time // a done key's end, read once from its history +} + +// Build is one done build of a module, as its key holds it, with when it ended. +type Build struct { + Module, FP, Key string + At time.Time + Artifacts []delivery.BuiltArtifact + // Manifest is the digest of the manifest it built, in the indexer's `manifests`. + Manifest string +} + +// LeaseAway says the controller's lease names a holder that is no controller: then the seat sweeps. +func (h *Holder) LeaseAway() bool { + raw, found, err := h.Bus.Get(LeaseRead, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, found, err) + return away +} + +// Builds is every done build the builder's keys hold, each with its end. +func (h *Holder) Builds() ([]Build, error) { + keys, err := h.Bus.Keys(BuildsRead) + if err != nil { + return nil, fmt.Errorf("the builder's builds: %w", err) + } + var out []Build + for _, k := range keys { + module, fp, err := delivery.ParseBuilderBuildKey(k) + if err != nil { + continue + } + raw, found, err := h.Bus.Get(BuildsRead, k) + if err != nil { + return nil, fmt.Errorf("the builder's build %s: %w", k, err) + } + if !found { + continue + } + b, err := delivery.DecodeBuilderBuild(raw) + if err != nil || b.State != delivery.BuildDone { + continue + } + at, err := h.endOf(k) + if err != nil { + return nil, err + } + out = append(out, Build{Module: module, FP: fp, Key: k, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest}) + } + return out, nil +} + +// endOf is when a done key was last written done, read once from its history: a done key is not written again. +func (h *Holder) endOf(key string) (time.Time, error) { + h.mu.Lock() + at, known := h.ended[key] + h.mu.Unlock() + if known { + return at, nil + } + revs, err := h.Bus.History(BuildsRead, key) + if err != nil { + return time.Time{}, fmt.Errorf("the history of the builder's build %s: %w", key, err) + } + for _, r := range revs { + if b, err := delivery.DecodeBuilderBuild(r.Value); err == nil && b.State == delivery.BuildDone && r.At.After(at) { + at = r.At + } + } + h.mu.Lock() + if h.ended == nil { + h.ended = map[string]time.Time{} + } + h.ended[key] = at + h.mu.Unlock() + return at, nil +} + +// Want is one node's want for a module. +type Want struct{ Node, Module, FP string } + +// Wants is every want that names a fingerprint. +func (h *Holder) Wants() ([]Want, error) { + keys, err := h.Bus.Keys(WantsRead) + if err != nil { + return nil, fmt.Errorf("the wants: %w", err) + } + var out []Want + for _, k := range keys { + node, module, ok := wantKey(k) + if !ok { + continue + } + raw, found, err := h.Bus.Get(WantsRead, k) + if err != nil { + return nil, fmt.Errorf("the want %s: %w", k, err) + } + var w delivery.Want + if !found || json.Unmarshal(raw, &w) != nil || w.Removed || w.FP == "" { + continue + } + out = append(out, Want{Node: node, Module: module, FP: w.FP}) + } + return out, nil +} + +// wantKey reads `want..`; the node's own key and a hold are not a module's want. +func wantKey(key string) (node, module string, ok bool) { + parts := strings.Split(key, ".") + if len(parts) != 3 || parts[0] != "want" || parts[2] == "node" { + return "", "", false + } + return parts[1], parts[2], true +} + +// Ref is one artifact the mesh made, as the keep set judges it. +type Ref struct { + Reference string + Kind string // image or archive + Builds []Build + Why []string + Kept bool + // At is its newest build's end: the oldest go first. + At time.Time +} + +// KeepSet judges every artifact the done builds name in the store: kept, and why, or eligible. Packages and +// references into no store of the mesh's are no business of the store's. +func KeepSet(builds []Build, wants []Want) map[string]*Ref { + keeps := keptBuilds(builds, wants) + out := map[string]*Ref{} + for _, b := range builds { + for _, a := range b.Artifacts { + reference := catalogue.Recorded(a.Reference) + if _, ours := catalogue.InArtifactStore(reference); !ours || a.Kind == "package" { + continue + } + r := out[reference] + if r == nil { + _, _, kind := splitReference(reference) + r = &Ref{Reference: reference, Kind: kind} + out[reference] = r + } + r.Builds = append(r.Builds, b) + r.keptBy(b, keeps[b.Key]) + } + } + return out +} + +// keptBy adds a build naming r, kept for why (none: eligible). +func (r *Ref) keptBy(b Build, why []string) { + if b.At.After(r.At) { + r.At = b.At + } + if len(why) > 0 { + r.Kept = true + for _, w := range why { + if !contains(r.Why, w) { + r.Why = append(r.Why, w) + } + } + } +} + +// keptBuilds is, by build key, why each kept build is kept: one of its module's five newest, or a want names it. +func keptBuilds(builds []Build, wants []Want) map[string][]string { + wanted := map[string][]string{} + for _, w := range wants { + wanted[w.Module+" "+w.FP] = append(wanted[w.Module+" "+w.FP], w.Node) + } + byModule := map[string][]Build{} + for _, b := range builds { + byModule[b.Module] = append(byModule[b.Module], b) + } + keeps := map[string][]string{} // build key → why + for module, bs := range byModule { + sort.Slice(bs, func(i, j int) bool { + if !bs[i].At.Equal(bs[j].At) { + return bs[i].At.After(bs[j].At) + } + return bs[i].Key > bs[j].Key + }) + for i, b := range bs { + if i < KeptBuilds { + keeps[b.Key] = append(keeps[b.Key], fmt.Sprintf("one of the %d newest builds of %s", KeptBuilds, module)) + } + if nodes := wanted[module+" "+b.FP]; len(nodes) > 0 { + sort.Strings(nodes) + keeps[b.Key] = append(keeps[b.Key], "a want names it on "+strings.Join(nodes, ", ")) + } + } + } + return keeps +} + +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +// inStore says whether the listing holds a reference: an image's manifest, an archive's blob. Unknown without a listing. +func inStore(v *View, reference string) (held, known bool) { + if v == nil || v.L == nil { + return false, false + } + repo, digest, kind := splitReference(reference) + switch kind { + case "image": + r := v.L.Repos[repo] + return r != nil && r.Revisions[digest], true + case "archive": + _, has := v.L.Blobs[digest] + return has, true + } + return false, false +} + +// judged is the keep set now, with the listing it was read against. +type judged struct { + refs map[string]*Ref + view *View + wants []Want + built []Build + // mirrors are the copies in the store's mirror repositories, by reference: recorded ones also in refs. + mirrors map[string]*Ref + // mirrorsHeld says why no copy is eligible now, when one is not: a kept build whose bases the log does not name. + mirrorsHeld string +} + +func (h *Holder) judge(ctx context.Context) (*judged, error) { + builds, err := h.Builds() + if err != nil { + return nil, err + } + wants, err := h.Wants() + if err != nil { + return nil, err + } + j := &judged{refs: KeepSet(builds, wants), wants: wants, built: builds} + if h.Listed != nil { + if j.view, err = h.Listed(ctx); err != nil { + h.logf("the store's listing could not be read, so nothing is said gone: %v", err) + j.view = nil + } + } + if j.view != nil { + against, err := h.against() + if err != nil { + h.logf("the builder's log could not be read, so every copy is kept as unrecorded: %v", err) + against = nil + } + j.mirrors, j.mirrorsHeld = Mirrors(j.view, builds, keptBuilds(builds, wants), against) + for r, m := range j.mirrors { + if len(m.Builds) > 0 { + j.refs[r] = m + } + } + } + return j, nil +} + +// against is what each build the builder's log says ended stood on, by build key: the bases' references. +func (h *Holder) against() (map[string][]string, error) { + entries, err := h.Bus.LogSince(delivery.BuildLogRead, h.now().Add(-delivery.BuilderLogDays*24*time.Hour)) + if err != nil { + return nil, err + } + out := map[string][]string{} + for _, raw := range entries { + e, err := delivery.DecodeBuildLogEntry(raw.Value) + if err != nil || e.Event != delivery.BuildEndedEvent { + continue + } + key, err := delivery.BuilderBuildKey(e.Module, e.FP) + if err != nil { + continue + } + out[key] = e.Against + } + return out, nil +} + +// Mirrors judges each copy in the store's mirror repositories (upstream//, /on-; ADR +// 0257) by what the builder's log says each build stood on (ADR 0571 clause 1): named by a kept build, kept; named +// only by builds no longer kept, eligible; named by no build the log holds, unrecorded and never removed (ADR 0189 +// §10). While a kept build's bases are not in the log (it ended before the log's 30 days), no copy is eligible: what +// it stood on cannot be told, and held says so. +func Mirrors(v *View, builds []Build, kept map[string][]string, against map[string][]string) (map[string]*Ref, string) { + out := map[string]*Ref{} + for _, name := range v.L.RepoNames() { + if !mirrorRepository.MatchString(name) { + continue + } + for d := range v.L.Repos[name].Revisions { + r := catalogue.ArtifactStoreScheme + name + "@" + d + out[r] = &Ref{Reference: r, Kind: "image"} + } + } + held := "" + for _, b := range builds { + bases, logged := against[b.Key] + if !logged { + if len(kept[b.Key]) > 0 && held == "" { + held = "the builder's log does not say what the kept build " + b.Key + " stood on, so every copy is kept" + } + continue + } + for _, base := range bases { + if m := out[catalogue.Recorded(base)]; m != nil { + m.Builds = append(m.Builds, b) + why := kept[b.Key] + if len(why) > 0 { + why = []string{"a kept build stood on it: " + b.Key} + } + m.keptBy(b, why) + } + } + } + if held != "" { + for _, m := range out { + if len(m.Builds) > 0 && !m.Kept { + m.Kept, m.Why = true, []string{held} + } + } + } + return out, held +} + +// Artifacts is the seat's `artifacts`, in the controller's answer's shape (Records): every reference the mesh made that +// lives in the store, kept with why or eligible, and with collected the ones the store no longer holds. +func (h *Holder) Artifacts(repository string) (*Records, error) { + j, err := h.judge(context.Background()) + if err != nil { + return nil, err + } + r := j.records(repository, true) + if d, err := h.Door(); err == nil { + r.Store = d.Address() + } + return r, nil +} + +func (j *judged) records(repository string, collected bool) *Records { + r := &Records{KeptBuilds: KeptBuilds, Counts: map[string]int{"kept": 0, "eligible": 0, "collected": 0}, CollectedAsked: collected} + for _, ref := range j.sorted() { + state := "eligible" + if ref.Kept { + state = "kept" + } + if held, known := inStore(j.view, ref.Reference); known && !held { + state = "collected" + if ref.Kept { + state = "kept" // a kept one gone is said missing by collect, never let go of + } + } + r.Counts[state]++ + if (repository != "" && !strings.HasPrefix(strings.TrimPrefix(ref.Reference, catalogue.ArtifactStoreScheme), repository)) || + (state == "collected" && !collected) { + continue + } + rec := Record{Reference: ref.Reference, Kind: ref.Kind, State: state, Why: ref.Why} + fill(&rec) + r.References = append(r.References, rec) + } + if j.view == nil { + r.Note = "the store's listing could not be read, so a reference it no longer holds is not said collected" + } + return r +} + +// sorted is the references oldest first, then by reference. +func (j *judged) sorted() []*Ref { + out := make([]*Ref, 0, len(j.refs)) + for _, r := range j.refs { + out = append(out, r) + } + sort.Slice(out, func(a, b int) bool { + if !out[a].At.Equal(out[b].At) { + return out[a].At.Before(out[b].At) + } + return out[a].Reference < out[b].Reference + }) + return out +} + +// Collect is the seat's `collect`, in the controller's answer's shape (CollectAnswer): a dry run unless confirm; with +// confirm and why, a sweep that also lets indexes go with their platform manifests. +func (h *Holder) Collect(why string, confirm bool, most int) (*CollectAnswer, error) { + if confirm && strings.TrimSpace(why) == "" { + return nil, errors.New("a confirmed collect needs why: nothing was let go of") + } + if most <= 0 { + most = MostPerSweep + } + if most > MostAtAll { + return nil, fmt.Errorf("collect lets go of at most %d at once, not %d", MostAtAll, most) + } + if confirm && !h.LeaseAway() { + return nil, errors.New("the controller holds its lease, and its take-in sweeps the store until it hands it away: " + + "this seat lets nothing go before then (novox/hq ADR 0571 clause 2); ask the controller's collect") + } + return h.Sweep(context.Background(), Sweeping{Confirmed: confirm, DryRun: !confirm, Most: most, Why: why, By: "collect"}) +} + +// Sweeping is one sweep's asks. +type Sweeping struct { + // Confirmed is a collect a person confirmed: an index goes with its platform manifests. + Confirmed bool + // DryRun lets nothing go and holds nothing: what would go is listed. + DryRun bool + Most int + Why string + // By names what asked: collect, or the build whose end it follows. + By string +} + +// Sweep holds every kept archive, then lets go of what is eligible, oldest first, within the bounds, recording each +// deletion and saying artifacts-gone. +func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) { + h.sweep.Lock() + defer h.sweep.Unlock() + j, err := h.judge(ctx) + if err != nil { + return nil, err + } + a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}} + var keptArchives, keptImages, eligible []string + for _, r := range j.sorted() { + switch { + case r.Kept && r.Kind == "archive": + keptArchives = append(keptArchives, r.Reference) + case r.Kept: + keptImages = append(keptImages, r.Reference) + default: + if held, known := inStore(j.view, r.Reference); known && !held { + continue // collected already + } + eligible = append(eligible, r.Reference) + } + } + a.KeptArchives, a.Eligible = len(keptArchives), len(eligible) + if s.DryRun { + for i, r := range eligible { + if i >= s.Most { + a.Left = len(eligible) - i + break + } + a.WouldLetGo = append(a.WouldLetGo, r) + } + return a, nil + } + door, err := h.Door() + if err != nil { + return nil, err + } + a.Store = door.Address() + within, stop := context.WithTimeout(ctx, SweepBudget) + defer stop() + // Hold before letting go (ADR 0251 §8): a kept archive that cannot be held stops the sweep before any delete. + var missing []string + for _, r := range keptArchives { + wrote, err := door.Hold(within, r) + switch { + case err == nil: + if wrote { + a.HoldersWritten++ + } + case errors.Is(err, registry.Gone): + missing = append(missing, r) + case errors.Is(err, registry.ErrNotOurs): + default: + a.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: holding %s: %v", r, err) + a.Left = len(eligible) + return a, nil + } + } + a.Held, a.Missing = len(keptArchives)-len(missing), len(missing) + h.sayGone(j, missing, "a kept archive the store does not hold") + if s.Confirmed { + if err := door.SpareKept(within, keptImages, eligible); err != nil { + a.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err) + a.Left = len(eligible) + return a, nil + } + } + for i, r := range eligible { + if i >= s.Most || within.Err() != nil { + a.Left = len(eligible) - i + if i >= s.Most { + a.Stopped = fmt.Sprintf("asked about %d, the most one sweep asks", s.Most) + } else { + a.Stopped = fmt.Sprintf("ran out of its %s", SweepBudget) + } + break + } + if !s.Confirmed { + // An index waits for a confirmed collect (ADR 0251 §7, ADR 0257 §4). + index, err := door.IsIndex(within, r) + if err != nil && !errors.Is(err, registry.Gone) && !errors.Is(err, registry.ErrNotOurs) { + a.Stopped = fmt.Sprintf("the store could not say whether %s is an index, so nothing more was asked: %v", r, err) + a.Left = len(eligible) - i + break + } + if index { + a.Skipped++ + continue + } + } + err := door.LetGo(within, r) + switch { + case err == nil || errors.Is(err, registry.Gone): + case errors.Is(err, registry.ErrNotOurs): + a.Skipped++ + continue + default: + a.Stopped = fmt.Sprintf("the store refused to let go of %s, so nothing more was asked: %v", r, err) + a.Left = len(eligible) - i + } + if a.Stopped != "" { + break + } + // Recorded in the hand-act log, each one (ADR 0565 §10; the operator's "option A"). + why := s.Why + if why == "" { + why = "eligible: no want names it and it is not among its module's " + fmt.Sprint(KeptBuilds) + " newest builds" + } + if _, err := h.Acts.Record(handacts.Request{What: "let go of " + r + " (" + s.By + ")", Why: why, + Cause: "artifact-store-sweep", Verb: "collect"}); err != nil { + h.logf("%s was let go of and could not be recorded in the hand-act log: %v", r, err) + } + a.LetGo = append(a.LetGo, r) + } + h.sayGone(j, a.LetGo, "let go of") + return a, nil +} + +// sayGone says artifacts-gone for each build that named one of refs: the builder builds a key again whose artifact is gone. +func (h *Holder) sayGone(j *judged, refs []string, how string) { + byKey := map[string]*delivery.ArtifactsGone{} + var keys []string + for _, r := range refs { + ref := j.refs[r] + if ref == nil { + continue + } + for _, b := range ref.Builds { + g := byKey[b.Key] + if g == nil { + g = &delivery.ArtifactsGone{Module: b.Module, Key: b.Key, Fingerprint: b.FP, At: h.now().UTC()} + byKey[b.Key] = g + keys = append(keys, b.Key) + } + g.Gone = append(g.Gone, r) + } + } + sort.Strings(keys) + for _, k := range keys { + raw, err := delivery.EncodeArtifactsGone(*byKey[k]) + if err == nil { + err = h.Bus.Publish(GoneSubject, json.RawMessage(raw)) + } + if err != nil { + h.logf("%s: %s %s, and artifacts-gone could not be said: %v", k, how, strings.Join(byKey[k].Gone, ", "), err) + } + } +} + +// AfterBuild is the sweep at a build's end: only once the lease names no controller. +func (h *Holder) AfterBuild(key string) { + if !h.LeaseAway() { + return + } + a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key}) + if err != nil { + h.logf("the sweep after %s: %v", key, err) + return + } + if len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0 { + h.logf("the sweep after %s let go of %d (%d left, %d missing): %s", key, len(a.LetGo), a.Left, a.Missing, a.Stopped) + } +} + +// Held is the seat's `held`: each reference, held, gone or unknown with why. Holds and writes nothing. +func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) { + if len(refs) == 0 { + return delivery.HeldAnswer{}, errors.New("held judges the references given, and none was") + } + if len(refs) > 64 { + return delivery.HeldAnswer{}, fmt.Errorf("held judges at most 64 references at once, and %d were given", len(refs)) + } + for _, r := range refs { + if _, ours := catalogue.InArtifactStore(r); !ours { + return delivery.HeldAnswer{}, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r) + } + } + door, err := h.Door() + if err != nil { + return delivery.HeldAnswer{}, err + } + asking, stop := context.WithTimeout(context.Background(), 20*time.Second) + defer stop() + a := delivery.HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}} + for _, r := range refs { + switch err := door.Holds(asking, r); { + case err == nil: + a.Held = append(a.Held, r) + case errors.Is(err, registry.Gone): + a.Gone = append(a.Gone, r) + default: + if a.Unknown == nil { + a.Unknown = map[string]string{} + } + a.Unknown[r] = err.Error() + } + } + return a, nil +} + +// Image is one image a machine's wants and haves name, as docker reads the answer. +type Image struct { + Image string `json:"image"` + Resources []string `json:"resources"` + In []string `json:"in"` +} + +// Images is the seat's `images`, the controller's answer's shape: every image one machine's wants (in: declared) and +// haves (in: sent) name, by the builds their fingerprints name, each with the module naming it. +type Images struct { + Node string `json:"node"` + Images []Image `json:"images"` + SentKnown bool `json:"sent_known"` +} + +// ImagesOf answers `images` for node. +func (h *Holder) ImagesOf(node string, address string) (Images, error) { + out := Images{Node: node, Images: []Image{}} + wants, err := h.Wants() + if err != nil { + return out, err + } + byImage := map[string]*Image{} + add := func(module, fp, in string) error { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return nil + } + raw, found, err := h.Bus.Get(BuildsRead, key) + if err != nil || !found { + return err + } + b, err := delivery.DecodeBuilderBuild(raw) + if err != nil { + return nil + } + images := []string{} + for _, a := range b.Artifacts { + if (a.Kind == "image" || a.Kind == "upstream") && a.Reference != "" { + images = append(images, catalogue.Routed(catalogue.Recorded(a.Reference), address)) + } + } + pinned, err := h.Pinned(b.Manifest) + if err != nil { + return err + } + for _, image := range append(images, pinned...) { + i := byImage[image] + if i == nil { + i = &Image{Image: image} + byImage[image] = i + } + if !contains(i.Resources, module) { + i.Resources = append(i.Resources, module) + } + if !contains(i.In, in) { + i.In = append(i.In, in) + } + } + return nil + } + for _, w := range wants { + if w.Node == node { + if err := add(w.Module, w.FP, "declared"); err != nil { + return out, err + } + } + } + keys, err := h.Bus.Keys(HavesRead) + if err != nil { + return out, fmt.Errorf("the haves: %w", err) + } + for _, k := range keys { + n, module, isNode, err := delivery.ParseHaveKey(k) + if err != nil || isNode || n != node { + continue + } + raw, found, err := h.Bus.Get(HavesRead, k) + if err != nil { + return out, err + } + var have delivery.Have + if !found || json.Unmarshal(raw, &have) != nil || have.FP == "" { + continue + } + out.SentKnown = true + if err := add(module, have.FP, "sent"); err != nil { + return out, err + } + } + for _, i := range byImage { + sort.Strings(i.Resources) + sort.Strings(i.In) + out.Images = append(out.Images, *i) + } + sort.Slice(out.Images, func(a, b int) bool { return out.Images[a].Image < out.Images[b].Image }) + return out, nil +} + +// Pinned is every container image a manifest names by itself, by digest (`/@sha256:…`), read from the +// indexer's manifests: what a definition pins outright rather than through a build's artifact. A placeholder is not one. +func (h *Holder) Pinned(manifest string) ([]string, error) { + if manifest == "" { + return nil, nil + } + key, err := delivery.ManifestsKey(manifest) + if err != nil { + return nil, nil + } + raw, found, err := h.Bus.Get(ManifestsRead, key) + if err != nil || !found { + return nil, err + } + m, err := delivery.DecodeManifest(raw) + if err != nil { + return nil, nil + } + var parsed struct { + Resources []map[string]any `json:"resources"` + } + if json.Unmarshal([]byte(m.JSON), &parsed) != nil { + return nil, nil + } + var out []string + for _, r := range parsed.Resources { + image, _ := r["image"].(string) + if fmt.Sprint(r["type"]) == "container" && strings.Contains(image, "@sha256:") && !strings.Contains(image, "${") && + !contains(out, image) { + out = append(out, image) + } + } + return out, nil +} + +// inStoreAt is the store's reference of an image its address names, "" for an image of another registry (the +// controller's imageInStore). +func inStoreAt(image, address string) string { + if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") { + return "" + } + r := catalogue.Recorded(image) + if _, ours := catalogue.InArtifactStore(r); !ours { + return "" + } + return r +} + +// Observe writes S25's findings (image-missing): each want whose build names an image the store does not hold, on the +// want's machine, and deletes each finding that no longer holds. Nothing is said without a listing. +func (h *Holder) Observe(ctx context.Context) error { + j, err := h.judge(ctx) + if err != nil { + return err + } + if j.view == nil { + return nil + } + address := "" + if d, err := h.Door(); err == nil { + address = d.Address() + } + builds := map[string]Build{} + for _, b := range j.built { + builds[b.Module+" "+b.FP] = b + } + now := map[string]health.Finding{} + for _, w := range j.wants { + b, ok := builds[w.Module+" "+w.FP] + if !ok { + continue + } + var refs []string + for _, a := range b.Artifacts { + if a.Kind == "image" { + refs = append(refs, catalogue.Recorded(a.Reference)) + } + } + // And what its manifest pins by itself under the store's address, as the controller judged S25. + if pinned, err := h.Pinned(b.Manifest); err == nil && address != "" { + for _, image := range pinned { + if r := inStoreAt(image, address); r != "" { + refs = append(refs, r) + } + } + } + for _, reference := range refs { + if held, known := inStore(j.view, reference); !known || held { + continue + } + f := imageMissing(w.Node, w.Module, reference, h.now()) + now[f.Key] = f + break + } + } + keys, err := h.Bus.Keys(FindingsState) + if err != nil { + return fmt.Errorf("the findings: %w", err) + } + for _, k := range keys { + if _, still := now[k]; !still { + if err := h.Bus.Delete(FindingsState, k); err != nil { + return err + } + } + } + for k, f := range now { + raw, found, err := h.Bus.Get(FindingsState, k) + if err != nil { + return err + } + if found { + if was, err := health.DecodeFinding(raw); err == nil && was.Summary == f.Summary { + continue // written only when what it says changes + } + } + if _, err := health.EncodeFinding(f); err != nil { + return err + } + if err := h.Bus.Put(FindingsState, k, f); err != nil { + return err + } + } + return nil +} + +// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages). +func imageMissing(node, module, image string, at time.Time) health.Finding { + id := module + "." + node + return health.Finding{V: health.FindingVersion, Key: health.FindingKey(health.ScopeModule, id, ImageMissing), Kind: ImageMissing, + Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node}, + Severity: health.Warning, Resolver: health.ResolverOperator, + Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+ + "not hold (the evidence names it); the running container is untouched", module, node), + Headline: capital(module) + " is not updated on " + node, + Explanation: fmt.Sprintf("The new version of %s names a container image that the mesh's store does not hold, so it "+ + "was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", module, node, node), + Needs: fmt.Sprintf("build %s again; the details name the image.", module), + Resolved: "Resolved: the store holds the image of " + module + " again", + Evidence: []health.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}}, + Source: "S25", Observer: Module, At: at.UTC()} +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func (h *Holder) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *Holder) logf(format string, a ...any) { + if h.Logf != nil { + h.Logf(format, a...) + } +} + +// Mirror is one copy as the seat's `mirrors` answers it. +type Mirror struct { + Reference string `json:"reference"` + State string `json:"state"` + Why []string `json:"why,omitempty"` + StoodOn []string `json:"stood_on_by,omitempty"` +} + +// MirrorsAnswer is the seat's `mirrors`: every copy, and why none is eligible when a kept build's bases are unknown. +type MirrorsAnswer struct { + Copies []Mirror `json:"copies"` + Held string `json:"held,omitempty"` +} + +// MirrorsNow is every copy in the store's mirror repositories as the keep set judges it now. +func (h *Holder) MirrorsNow() (MirrorsAnswer, error) { + j, err := h.judge(context.Background()) + if err != nil { + return MirrorsAnswer{}, err + } + if j.view == nil { + return MirrorsAnswer{}, errors.New("the store's listing could not be read, so no copy can be named") + } + out := MirrorsAnswer{Copies: []Mirror{}, Held: j.mirrorsHeld} + for _, r := range j.mirrors { + m := Mirror{Reference: r.Reference, State: "unrecorded", Why: r.Why} + switch { + case r.Kept: + m.State = "kept" + case len(r.Builds) > 0: + m.State = "eligible" + } + for _, b := range r.Builds { + m.StoodOn = append(m.StoodOn, b.Key) + } + out.Copies = append(out.Copies, m) + } + sort.Slice(out.Copies, func(a, b int) bool { return out.Copies[a].Reference < out.Copies[b].Reference }) + return out, nil +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go new file mode 100644 index 00000000..f1264823 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -0,0 +1,260 @@ +//go:build integration + +package main + +// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done +// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch +// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus, +// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's +// door is in memory: its HTTP is the copied registry package's, tested there. + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// realBus is the holder's Bus over a real nats-server: each read state a bucket named as the runtime names it. +type realBus struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + // log is the builder's log as the runtime's log verbs would read it (mesh-tools adds them; the sdk's own read of a + // log is tested in mesh-sdk's process package). + log []stdio.LogEntry +} + +func (b realBus) LogSince(_ string, at time.Time) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + for _, e := range b.log { + if !e.At.Before(at) { + out = append(out, e) + } + } + return out, nil +} + +func bucketOf(state string) string { return strings.Replace(state, ".", "_", 1) } + +func (b realBus) kv(state string) (jetstream.KeyValue, error) { + return b.js.KeyValue(b.ctx, bucketOf(state)) +} + +func (b realBus) Keys(state string) ([]string, error) { + kv, err := b.kv(state) + if err != nil { + return nil, err + } + keys, err := kv.Keys(b.ctx) + if err == jetstream.ErrNoKeysFound { + return nil, nil + } + return keys, err +} + +func (b realBus) Get(state, key string) ([]byte, bool, error) { + kv, err := b.kv(state) + if err != nil { + return nil, false, err + } + e, err := kv.Get(b.ctx, key) + if err == jetstream.ErrKeyNotFound { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + +func (b realBus) History(state, key string) ([]stdio.StateRevision, error) { + kv, err := b.kv(state) + if err != nil { + return nil, err + } + h, err := kv.History(b.ctx, key) + if err != nil { + return nil, err + } + var out []stdio.StateRevision + for _, e := range h { + out = append(out, stdio.StateRevision{Key: key, Value: e.Value(), Revision: e.Revision(), At: e.Created()}) + } + return out, nil +} + +func (b realBus) Put(state, key string, value any) error { + kv, err := b.kv(state) + if err != nil { + return err + } + raw, _ := json.Marshal(value) + _, err = kv.Put(b.ctx, key, raw) + return err +} + +func (b realBus) Delete(state, key string) error { + kv, err := b.kv(state) + if err != nil { + return err + } + return kv.Delete(b.ctx, key) +} + +func (b realBus) Publish(subject string, body any) error { + raw, _ := json.Marshal(body) + return b.nc.Publish(subject, raw) +} + +func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil { + t.Fatal(err) + } + } + bus := realBus{ctx: ctx, nc: nc, js: js} + + // The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing. + mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}) + bus.log = memb.log + for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} { + keys, _ := memb.Keys(s) + for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus + raw, _, _ := memb.Get(s, k) + if err := bus.Put(s, k, json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + } + } + time.Sleep(20 * time.Millisecond) // each build's end is its key's time on the bus: the seventh the newest + + // The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject. + var mu sync.Mutex + var acts, gone []string + if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) { + mu.Lock() + acts = append(acts, string(m.Data)) + mu.Unlock() + _ = m.Respond([]byte(`{"ok":true}`)) + }); err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe(GoneSubject, func(m *nats.Msg) { + mu.Lock() + gone = append(gone, string(m.Data)) + mu.Unlock() + }); err != nil { + t.Fatal(err) + } + h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf} + h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second) + if err != nil { + return nil, err + } + return m.Data, nil + }} + + // A build's end heard from the builds bucket, as follow hands it over. + kv, _ := bus.kv(BuildsRead) + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + ended := func() { + keys, _ := memb.Keys(BuildsRead) + raw, _, _ := memb.Get(BuildsRead, keys[6]) + if err := bus.Put(BuildsRead, keys[6], json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + select { + case e := <-w.Updates(): + h.AfterBuild(e.Key()) + case <-ctx.Done(): + t.Fatal("the build's end was not heard") + } + } + + // While the lease names the controller, a build's end lets nothing go. + ended() + if len(door.asked) != 0 { + t.Fatalf("before the lease write the store was asked %v", door.asked) + } + // The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved + // past the five newest and no want names (the want keeps the 1st), recording each deletion. + if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil { + t.Fatal(err) + } + ended() + deadline := time.Now().Add(5 * time.Second) + for { + mu.Lock() + n := len(gone) + mu.Unlock() + if n > 0 || time.Now().After(deadline) { + break + } + time.Sleep(20 * time.Millisecond) + } + mu.Lock() + defer mu.Unlock() + if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) || + !strings.Contains(acts[2], mirror(1)) { + t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked) + } + if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) { + t.Fatalf("artifacts-gone said %v", gone) + } + + // S25 through the real findings bucket: letta's wanted image missing from the store is a finding there. + view, _ := mem.Listed(ctx) + repo, digest, _ := splitReference(imageRef("letta", 1)) + delete(view.L.Repos[repo].Revisions, digest) + if err := h.Observe(ctx); err != nil { + t.Fatal(err) + } + if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" { + t.Fatalf("the findings are %v", keys) + } +} + +// findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings. +type findingsAt struct{ realBus } + +func (f findingsAt) own(state string) string { + if state == FindingsState { + return Module + "." + state + } + return state +} +func (f findingsAt) Keys(state string) ([]string, error) { return f.realBus.Keys(f.own(state)) } +func (f findingsAt) Get(state, key string) ([]byte, bool, error) { + return f.realBus.Get(f.own(state), key) +} +func (f findingsAt) Put(state, key string, value any) error { + return f.realBus.Put(f.own(state), key, value) +} +func (f findingsAt) Delete(state, key string) error { return f.realBus.Delete(f.own(state), key) } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go new file mode 100644 index 00000000..f1d6b7a1 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -0,0 +1,351 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" + + "artifactstoretools/internal/registry" +) + +// memBus is the holder's bus in memory: states by name, each key's history, and what was published. +type memBus struct { + mu sync.Mutex + states map[string]map[string][]byte + history map[string][]stdio.StateRevision + log []stdio.LogEntry + published []string +} + +func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + for _, e := range b.log { + if read == delivery.BuildLogRead && !e.At.Before(at) { + out = append(out, e) + } + } + return out, nil +} + +func (b *memBus) Keys(state string) ([]string, error) { + b.mu.Lock() + defer b.mu.Unlock() + var out []string + for k := range b.states[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (b *memBus) Get(state, key string) ([]byte, bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + v, ok := b.states[state][key] + return v, ok, nil +} + +func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) { + return b.history[state+" "+key], nil +} + +func (b *memBus) Put(state, key string, value any) error { + raw, _ := json.Marshal(value) + b.mu.Lock() + defer b.mu.Unlock() + if b.states[state] == nil { + b.states[state] = map[string][]byte{} + } + b.states[state][key] = raw + return nil +} + +func (b *memBus) Delete(state, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.states[state], key) + return nil +} + +func (b *memBus) Publish(subject string, body any) error { + raw, _ := json.Marshal(body) + b.mu.Lock() + defer b.mu.Unlock() + b.published = append(b.published, subject+" "+string(raw)) + return nil +} + +// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked. +type memDoor struct { + holds map[string]bool + indexes map[string]bool + asked []string +} + +func (d *memDoor) Address() string { return "10.0.0.5:5000" } +func (d *memDoor) Hold(_ context.Context, r string) (bool, error) { + d.asked = append(d.asked, "hold "+r) + if !d.holds[r] { + return false, registry.Gone + } + return false, nil +} +func (d *memDoor) Holds(_ context.Context, r string) error { + if !d.holds[r] { + return registry.Gone + } + return nil +} +func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil } +func (d *memDoor) LetGo(_ context.Context, r string) error { + d.asked = append(d.asked, "let go "+r) + delete(d.holds, r) + return nil +} +func (d *memDoor) SpareKept(context.Context, []string, []string) error { + d.asked = append(d.asked, "spare") + return nil +} + +// mirror is a copy of an upstream base in the store, the n-th. +func mirror(n int) string { + return "artifact-store://upstream/docker.io/library/alpine@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) +} + +// pinnedImage is the image letta's manifest names by itself, at the store's address. +const pinnedImage = "10.0.0.5:5000/letta/pinned@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + +func digestOf(ref string) string { _, d, _ := splitReference(ref); return d } + +func fp(n int) string { return "src2:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) } + +func imageRef(module string, n int) string { + return "artifact-store://" + module + "/image@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) +} + +func archiveRef(module string, n int) string { + return "artifact-store://" + module + "/files/blobs/sha256:" + strings.Repeat(fmt.Sprintf("%x", (n+8)%16), 64) +} + +// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive; +// a want names the first on ace; the store lists them all and one image no build names; the lease as given. +func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) { + t.Helper() + day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}} + door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}} + listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}} + for n := 1; n <= 7; n++ { + key, _ := delivery.BuilderBuildKey("letta", fp(n)) + raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{ + {Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)}, + {Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)}, + {Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}}) + if err != nil { + t.Fatal(err) + } + _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + b.history[BuildsRead+" "+key] = []stdio.StateRevision{{Key: key, Value: raw, At: day.Add(time.Duration(n) * time.Hour)}} + for _, r := range []string{imageRef("letta", n), archiveRef("letta", n)} { + door.holds[r] = true + repo, digest, kind := splitReference(r) + if kind == "archive" { + listing.Blobs[digest] = 10 + } else { + listing.repo(repo).Revisions[digest] = true + } + } + } + listing.repo("letta/image").Revisions["sha256:"+strings.Repeat("e", 64)] = true // no build names it + listing.repo("letta/pinned").Revisions["sha256:"+strings.Repeat("c", 64)] = true + // Three copies of a base: letta's 2nd stood on the first, its 7th on the second, and no build on the third. + for n := 1; n <= 3; n++ { + repo, digest, _ := splitReference(mirror(n)) + listing.repo(repo).Revisions[digest] = true + door.holds[mirror(n)] = true + } + for n := 1; n <= 7; n++ { + e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c", + Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"} + switch n { + case 2: + e.Against = []string{mirror(1)} + case 7: + e.Against = []string{mirror(2)} + } + raw, _ := delivery.EncodeBuildLogEntry(e) + b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At}) + } + manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta", + JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` + + `{"id":"built","type":"container","image":"${artifact:image}"}]}`}) + mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64)) + _ = b.Put(ManifestsRead, mk, json.RawMessage(manifest)) + want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true}) + _ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want)) + if lease != nil { + _ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease) + } + var acts []string + h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, + Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil }, + Now: func() time.Time { return day.Add(48 * time.Hour) }, + Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + acts = append(acts, key+" "+string(raw)) + return json.RawMessage(`{}`), nil + }}} + h.Acts.LeaseAway = h.LeaseAway + return h, b, door, &acts +} + +// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while +// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound, +// leaves an index for a confirmed collect, records each deletion and says artifacts-gone. +func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) { + controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3} + handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"} + + h, _, _, _ := aMesh(t, controller) + r, err := h.Artifacts("") + if err != nil { + t.Fatal(err) + } + states := map[string]string{} + for _, x := range r.References { + states[x.Reference] = x.State + } + // letta's 3rd to 7th are the five newest; its 1st a want names on ace, however old; its 2nd is eligible; the package + // and the manifest no build names are no reference of the seat's. + for n, want := range map[int]string{1: "kept", 2: "eligible", 3: "kept", 7: "kept"} { + if states[imageRef("letta", n)] != want || states[archiveRef("letta", n)] != want { + t.Errorf("letta's build %d is %s and %s, not %s", n, states[imageRef("letta", n)], states[archiveRef("letta", n)], want) + } + } + // And the copies: the one letta's 2nd stood on is eligible with it, the 7th's kept; the third no build names. + if states[mirror(1)] != "eligible" || states[mirror(2)] != "kept" || states[mirror(3)] != "" { + t.Errorf("the copies are %s, %s and %q", states[mirror(1)], states[mirror(2)], states[mirror(3)]) + } + if m, err := h.MirrorsNow(); err != nil || len(m.Copies) != 3 || m.Copies[2].State != "unrecorded" || m.Held != "" { + t.Errorf("mirrors answered %+v: %v", m, err) + } + if len(r.References) != 16 || r.Counts["kept"] != 13 || r.Counts["eligible"] != 3 || r.Store != "10.0.0.5:5000" { + t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store) + } + + for _, c := range []struct { + name string + lease *buscompose.LeaseHolder + index bool + most int + letGo []string + left int + skipped int + unlog int + }{ + {name: "the lease names a controller", lease: controller}, + {name: "no lease key", lease: nil}, + {name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}}, + {name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1}, + {name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2}, + // A kept build the log no longer holds: what it stood on is unknown, so no copy goes. + {name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3}, + } { + t.Run(c.name, func(t *testing.T) { + h, b, door, acts := aMesh(t, c.lease) + door.indexes[imageRef("letta", 2)] = c.index + if c.unlog > 0 { + b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...) + } + if c.most == 0 { + h.AfterBuild("letta.x") + if _, err := h.Collect("tidy", true, 0); err == nil { + t.Error("a confirmed collect was let through while the controller sweeps") + } + if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 { + t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published) + } + return + } + a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"}) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 { + t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo) + } + // Every kept archive held before the first deletion. + first := -1 + for i, x := range door.asked { + if strings.HasPrefix(x, "let go ") && first < 0 { + first = i + } + } + if first != 6 { + t.Errorf("the store was asked %v: the six kept archives are held first", door.asked) + } + if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") || + !strings.Contains((*acts)[0], c.letGo[0]) { + t.Errorf("the hand acts recorded are %v", *acts) + } + if len(b.published) != 1 || !strings.HasPrefix(b.published[0], GoneSubject+" ") || + !strings.Contains(b.published[0], `"fingerprint":"`+fp(2)+`"`) { + t.Errorf("said %v", b.published) + } + }) + } +} + +// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state, +// written once and deleted once the store holds it again; images names what a machine's wants and haves name. +func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) { + h, b, _, _ := aMesh(t, nil) + view, _ := h.Listed(context.Background()) + repo, digest, _ := splitReference(imageRef("letta", 1)) + delete(view.L.Repos[repo].Revisions, digest) + h.Listed = func(context.Context) (*View, error) { return view, nil } + if err := h.Observe(context.Background()); err != nil { + t.Fatal(err) + } + key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing) + raw, found, _ := b.Get(FindingsState, key) + f, err := health.DecodeFinding(raw) + if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) { + t.Fatalf("the finding is %+v (%v, %v)", f, found, err) + } + view.L.Repos[repo].Revisions[digest] = true + if err := h.Observe(context.Background()); err != nil { + t.Fatal(err) + } + if _, found, _ := b.Get(FindingsState, key); found { + t.Fatal("the finding stayed once the store held the image again") + } + // An image the manifest pins by itself, under the store's address, is judged as the controller judged it. + delete(view.L.Repos["letta/pinned"].Revisions, "sha256:"+strings.Repeat("c", 64)) + if err := h.Observe(context.Background()); err != nil { + t.Fatal(err) + } + if raw, found, _ := b.Get(FindingsState, key); !found || !strings.Contains(string(raw), "letta/pinned@sha256:") { + t.Fatalf("the pinnedImage image missing is %s", raw) + } + + have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"}) + _ = b.Put(HavesRead, "ace.letta", json.RawMessage(have)) + images, err := h.ImagesOf("ace", "10.0.0.5:5000") + if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) || + fmt.Sprint(images.Images[0].In) != "[declared]" || fmt.Sprint(images.Images[1].In) != "[sent]" || + images.Images[2].Image != pinnedImage || fmt.Sprint(images.Images[2].In) != "[declared sent]" { + t.Fatalf("the images of ace are %+v: %v", images, err) + } +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index f833fecf..ea1f28e0 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -1,30 +1,38 @@ -// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, to-be 51): a process the node's -// runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its -// repositories, its size, and what the controller's records say of each manifest — read from the -// store's own files through the store's own container, and writing nothing. A collection is asked of -// the controller, which decides and records what it lets go of (ADR 0189). +// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the +// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its +// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container, +// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names +// no controller, and S25's image-missing. // -// **A module beside the store, not the store's own.** A module that provides the artifact store cannot -// also build an artifact: building publishes to the store, so the store would be needed to create -// itself, and the controller refuses that manifest. So these tools are a second module, assigned to the -// machine that holds the store. stdout is the protocol; what this bundle says, it says on stderr. +// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an +// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a +// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says +// on stderr. package main import ( "fmt" "os" + "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" ) +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + func main() { s := &Store{ Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"), Run: ExecRunner, UID: os.Getuid(), } + h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View, + Now: time.Now, Logf: logf} + h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway} + go follow(h) // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools. - if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil { + if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/seat.go b/modules/artifact-store-tools/cmd/artifact-store-tools/seat.go new file mode 100644 index 00000000..37491142 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/seat.go @@ -0,0 +1,122 @@ +package main + +// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named +// `artifact-store.` as the runtime serves a seat's verb, answering as the controller's verb answered: {output, +// ok, answer}. + +import ( + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf). +type verbAnswer struct { + Output string `json:"output"` + OK bool `json:"ok"` + Answer any `json:"answer"` +} + +// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors. +func SeatTools(h *Holder, s *Store) []stdio.Tool { + seat := catalogue.ArtifactSweepSeat + "." + return []stdio.Tool{ + {Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " + + "or with collected those it no longer holds.", + Run: func(args map[string]any) (any, error) { + r, err := h.Artifacts(text(args, "repository")) + if err != nil { + return nil, err + } + if text(args, "collected") != "true" { + refs := r.References[:0:0] + for _, x := range r.References { + if x.State != "collected" { + refs = append(refs, x) + } + } + r.References = refs + } + return verbAnswer{Output: fmt.Sprintf("%d kept, %d eligible, %d collected\n", r.Counts["kept"], + r.Counts["eligible"], r.Counts["collected"]), OK: true, Answer: r}, nil + }}, + {Name: seat + "held", Description: "Whether the store holds each reference given: held, gone or unknown with why.", + Run: func(args map[string]any) (any, error) { + a, err := h.Held(listOf(args["references"])) + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked\n", + a.Store, len(a.Held), len(a.Held)+len(a.Gone)+len(a.Unknown)), OK: true, Answer: a}, nil + }}, + {Name: seat + "collect", Description: "The sweep now: a dry run without confirm; with confirm and why, every kept " + + "archive held, then each eligible artifact let go of, each recorded in the hand-act log.", + Run: func(args map[string]any) (any, error) { + most, err := count(args, "most", MostPerSweep, 1, MostAtAll) + if err != nil { + return nil, err + } + a, err := h.Collect(text(args, "why"), text(args, "confirm") == "true", most) + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("let go of %d, would let go of %d, %d left\n", len(a.LetGo), + len(a.WouldLetGo), a.Left), OK: true, Answer: a}, nil + }}, + {Name: seat + "mirrors", Description: "The bases the mesh copied into its store, each kept (a kept build stood on " + + "it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood " + + "on it: never removed).", + Run: func(args map[string]any) (any, error) { + if text(args, "record") != "" { + return nil, fmt.Errorf("mirrors records nothing by hand on this seat: a copy is recorded by the builds the " + + "builder's log says stood on it (novox/hq ADR 0571 clause 1)") + } + m, err := h.MirrorsNow() + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("%d copies\n", len(m.Copies)), OK: true, Answer: m}, nil + }}, + {Name: seat + "images", Description: "Every container image one machine's wants and haves name.", + Run: func(args map[string]any) (any, error) { + node := text(args, "node") + if node == "" { + return nil, fmt.Errorf("images names the machine") + } + address := "" + if d, err := h.Door(); err == nil { + address = d.Address() + } + a, err := h.ImagesOf(node, address) + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("%s: %d images\n", node, len(a.Images)), OK: true, Answer: a}, nil + }}, + } +} + +// listOf is a list argument: a list, or one text separated by commas. +func listOf(v any) []string { + var out []string + add := func(s string) { + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" && !contains(out, p) { + out = append(out, p) + } + } + } + switch x := v.(type) { + case string: + add(x) + case []any: + for _, e := range x { + if s, ok := e.(string); ok { + add(s) + } + } + } + return out +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 72b46379..e0617a03 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -1,6 +1,8 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/catalogue" + "context" "encoding/json" "fmt" @@ -411,6 +413,11 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { var m struct { Tools []string `json:"tools"` Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -428,10 +435,30 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { if !reflect.DeepEqual(served, listed) { t.Fatalf("served %v, manifest %v", served, listed) } - want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule} + // ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's; + // the lease is the one controller state it reads (clause 10's exemption). + want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule} if !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes %v", m.Invokes) } + var verbs []string + for _, tl := range SeatTools(&Holder{}, &Store{}) { + verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+".")) + } + seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat) + var contract []string + for _, v := range seat.Serves { + contract = append(contract, v.Name) + } + if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) || + !reflect.DeepEqual(verbs, contract) { + t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract) + } + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead { + t.Errorf("reads the controller's %s", r) + } + } } func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) { diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go b/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go index 65d0350e..a6656750 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go @@ -11,9 +11,16 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) +// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR +// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape. +type Recorded interface { + Artifacts(repository string) (*Records, error) + Collect(why string, confirm bool, most int) (*CollectAnswer, error) +} + // Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect -// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes. -func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool { +// only reads too, unless it is a real run, and then it asks the seat's sweep, which decides and deletes. +func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool { ctx := context.Background() repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (/)"} return []stdio.Tool{ diff --git a/modules/artifact-store-tools/files/store-address b/modules/artifact-store-tools/files/store-address new file mode 100644 index 00000000..1bcadab2 --- /dev/null +++ b/modules/artifact-store-tools/files/store-address @@ -0,0 +1 @@ +${seat:mesh-artifact-store:reach} diff --git a/modules/artifact-store-tools/go.mod b/modules/artifact-store-tools/go.mod index 74ccd152..b2a35897 100644 --- a/modules/artifact-store-tools/go.mod +++ b/modules/artifact-store-tools/go.mod @@ -1,5 +1,22 @@ module artifactstoretools -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/artifact-store-tools/go.sum b/modules/artifact-store-tools/go.sum index b474419e..a2b8e018 100644 --- a/modules/artifact-store-tools/go.sum +++ b/modules/artifact-store-tools/go.sum @@ -1,2 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/artifact-store-tools/internal/registry/blob.go b/modules/artifact-store-tools/internal/registry/blob.go new file mode 100644 index 00000000..bedbfa38 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/blob.go @@ -0,0 +1,57 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" +) + +// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a +// take-in reads text from. +const blobBound = 64 << 20 + +// GetBlob is the archive a recorded reference names (`artifact-store:///blobs/sha256:`), read whole +// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the +// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold +// it; ErrNotOurs for a reference outside the store or one that is not an archive. +func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil { + return nil, err + } + if !archive { + return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference) + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil) + if err != nil { + return nil, err + } + response, err := s.client().Do(request) + if err != nil { + return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, Gone + default: + return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1)) + if err != nil { + return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err) + } + if len(body) > blobBound { + return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound) + } + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository, + digest, got) + } + return body, nil +} diff --git a/modules/artifact-store-tools/internal/registry/blob_test.go b/modules/artifact-store-tools/internal/registry/blob_test.go new file mode 100644 index 00000000..13487cc7 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/blob_test.go @@ -0,0 +1,44 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names. +func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) { + good := []byte("the archive") + sum := sha256.Sum256(good) + digest := "sha256:" + hex.EncodeToString(sum[:]) + liar := "sha256:" + strings.Repeat("0", 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar): + _, _ = w.Write(good) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + s := Store{Address: strings.TrimPrefix(server.URL, "http://")} + ctx := context.Background() + body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest) + if err != nil || string(body) != string(good) { + t.Fatalf("the archive: %q %v", body, err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("bytes of another digest are read: %v", err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) { + t.Fatalf("a blob the store does not hold: %v", err) + } + if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) { + t.Fatalf("a reference outside the store: %v", err) + } +} diff --git a/modules/artifact-store-tools/internal/registry/hold.go b/modules/artifact-store-tools/internal/registry/hold.go new file mode 100644 index 00000000..391e9c89 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/hold.go @@ -0,0 +1,345 @@ +package registry + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189). +// +// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and +// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs +// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what +// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a +// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a +// bare blob is unreferenced, so the first real collection would have deleted every archive the +// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The +// collector runs `--dry-run` until this is true. +// +// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty +// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The +// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its +// holder first, which is what lets the bytes go at the next collection. +// +// **Nothing a machine reads changes.** The recorded reference stays +// `artifact-store:////blobs/sha256:…`, and machines fetch the blob exactly as +// before. The holder is the store's bookkeeping, not a second way to reach anything. +// +// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest +// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can +// compute which manifest holds any archive from the reference it already has plus one HEAD for the +// size. No schema change, no second record that could disagree with the store. + +const ( + // mediaManifest is the type a holder is put and asked for as. + mediaManifest = "application/vnd.oci.image.manifest.v1+json" + // mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest + // whose only purpose is to name its layer. + mediaEmpty = "application/vnd.oci.empty.v1+json" + // emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec. + emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + // mediaArchive is the layer type an archive is held as. Every archive the builder publishes is + // `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant, + // not read from anywhere, because the holder must be recomputable from the reference alone. + mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip" +) + +// emptyConfig is the content emptyDigest names. +var emptyConfig = []byte("{}") + +// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a +// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well +// (measured 2026-09-28; internal/builder/registry.go says how that was found). +var manifestAccept = []string{ + mediaManifest, + "application/vnd.docker.distribution.manifest.v2+json", +} + +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +type holderManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` +} + +// Holder is the manifest that holds an archive in the store, and its digest. +// +// A pure function of the archive's digest and size: the same two in give the same bytes out, +// always, because `encoding/json` writes a struct's fields in their declared order and there is +// nothing here that varies by when or where it was composed. +func Holder(digest string, size int64) (body []byte, holder string) { + body, err := json.Marshal(holderManifest{ + SchemaVersion: 2, + MediaType: mediaManifest, + Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))}, + Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}}, + }) + if err != nil { + // Marshalling a struct of strings and integers cannot fail. + panic(err) + } + sum := sha256.Sum256(body) + return body, "sha256:" + hex.EncodeToString(sum[:]) +} + +// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one. +// +// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so +// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which +// is what lets the sweep run it over every kept archive on every build and so backfill the bare +// blobs published before holders existed (novox/hq issue 253). +// +// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a +// fact the caller reports rather than one this invents a remedy for. +func (s Store) Hold(ctx context.Context, reference string) (bool, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil || !archive { + return false, err + } + size, err := s.blobSize(ctx, repository, digest) + if err != nil { + return false, err + } + return s.HoldBlob(ctx, repository, digest, size) +} + +// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the +// question an operator needs answered with "none unheld" before the collector is let loose. +// +// An image answers true: it is its own manifest. An archive the store does not have answers Gone. +func (s Store) Held(ctx context.Context, reference string) (bool, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil { + return false, err + } + if !archive { + return true, nil + } + size, err := s.blobSize(ctx, repository, digest) + if err != nil { + return false, err + } + _, holder := Holder(digest, size) + return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...) +} + +// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is +// there already. Answers whether it wrote one. +// +// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size +// the store reports. Both arrive at the same holder, which is the point of composing it. +func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) { + if s.Address == "" { + return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest) + } + body, holder := Holder(digest, size) + there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...) + if err != nil { + return false, err + } + if there { + return false, nil + } + + // The config must be in the repository before a manifest naming it is accepted: a registry + // refuses a manifest whose blobs it cannot find there, which is the property that makes a + // holder mean something. + if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil { + return false, err + } + + // **By digest, never by tag.** A tag would be one more name to move and one more thing the + // collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that + // it pins by digest, and an untagged manifest is kept by plain collection. + request, err := http.NewRequestWithContext(ctx, http.MethodPut, + s.url(repository, "manifests", holder), bytes.NewReader(body)) + if err != nil { + return false, err + } + request.Header.Set("Content-Type", mediaManifest) + response, err := s.client().Do(request) + if err != nil { + return false, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusCreated { + said, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s", + repository, digest, response.Status, strings.TrimSpace(string(said))) + } + return true, nil +} + +// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes. +// +// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the +// collector would keep its bytes for ever on the strength of it — the sweep would record the +// archive collected while the disk said otherwise. Deleting the holder first and failing before +// the link goes leaves an unheld archive that the next sweep still offers, which is safe. +// +// A store that no longer has the blob answers Gone: without its size the holder cannot be named, +// and without the blob there is nothing left for a holder to keep. A store that never had a holder +// for it — an archive published before holders, never backfilled — answers 404 to the delete, and +// that is the outcome wanted. +func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error { + size, err := s.blobSize(ctx, repository, digest) + if err != nil { + return err + } + _, holder := Holder(digest, size) + err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder) + if err == Gone { + return nil + } + return err +} + +// archive reads a recorded reference into its repository and digest, and whether it is an archive +// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store. +func (s Store) archive(reference string) (repository, digest string, archive bool, err error) { + path, kept := catalogue.InArtifactStore(reference) + if !kept { + return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + if s.Address == "" { + return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return "", "", false, err + } + return repository, digest, kind == "blobs", nil +} + +// blobSize is how large the store says a blob is; Gone when it does not have it. +func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil) + if err != nil { + return 0, err + } + response, err := s.client().Do(request) + if err != nil { + return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return 0, Gone + default: + return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + // Read from the header rather than ContentLength: a HEAD's ContentLength is what the response + // says it would have sent, which Go reports faithfully, but a proxy in between is free to drop + // it, and the header is what the registry itself wrote. + if length := response.Header.Get("Content-Length"); length != "" { + if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 { + return n, nil + } + } + if response.ContentLength >= 0 { + return response.ContentLength, nil + } + return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest) +} + +// putBlob uploads a small blob unless the repository already has it: ask where, then put it there +// naming the digest — the registry's own two steps, the same the builder takes for an archive. +func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error { + if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil { + return err + } else if there { + return nil + } + start, err := http.NewRequestWithContext(ctx, http.MethodPost, + "http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil) + if err != nil { + return err + } + begun, err := s.client().Do(start) + if err != nil { + return fmt.Errorf("cannot start an upload to %s: %w", repository, err) + } + begun.Body.Close() + if begun.StatusCode != http.StatusAccepted { + return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository) + } + where := begun.Header.Get("Location") + if where == "" { + return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository) + } + if strings.HasPrefix(where, "/") { + where = "http://" + s.Address + where + } + separator := "?" + if strings.Contains(where, "?") { + separator = "&" + } + put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body)) + if err != nil { + return err + } + put.Header.Set("Content-Type", "application/octet-stream") + done, err := s.client().Do(put) + if err != nil { + return err + } + defer done.Body.Close() + if done.StatusCode != http.StatusCreated { + said, _ := io.ReadAll(io.LimitReader(done.Body, 4096)) + return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said))) + } + return nil +} + +// has is whether the store answers 200 for a HEAD at that URL. +func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return false, err + } + for _, media := range accept { + request.Header.Add("Accept", media) + } + response, err := s.client().Do(request) + if err != nil { + return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url) + } +} + +func (s Store) url(repository, kind, digest string) string { + return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest +} + +func (s Store) client() *http.Client { + if s.HTTP != nil { + return s.HTTP + } + return &http.Client{Timeout: 30 * time.Second} +} diff --git a/modules/artifact-store-tools/internal/registry/hold_test.go b/modules/artifact-store-tools/internal/registry/hold_test.go new file mode 100644 index 00000000..8a3a16fa --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/hold_test.go @@ -0,0 +1,268 @@ +package registry + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189). +// +// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a +// registry refuses — a blob whose digest does not match, a manifest whose digest does not match +// or whose blobs the repository does not have, a manifest asked for without an Accept naming its +// type. What is asserted is this side's decisions; the live test below asserts the registry's. + +type memRegistry struct { + mu sync.Mutex + blobs map[string][]byte // repository + "@" + digest + manifests map[string][]byte // repository + "@" + digest + writes []string // every PUT and DELETE, as "METHOD path" +} + +func digestOf(body []byte) string { + sum := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func (m *memRegistry) serve(t *testing.T) Store { + t.Helper() + m.blobs = map[string][]byte{} + m.manifests = map[string][]byte{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + path := strings.TrimPrefix(r.URL.Path, "/v2/") + if r.Method == http.MethodPut || r.Method == http.MethodDelete { + m.writes = append(m.writes, r.Method+" "+r.URL.Path) + } + switch { + case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"): + repository := strings.TrimSuffix(path, "/blobs/uploads/") + w.Header().Set("Location", "/upload/"+repository+"?state=x") + w.WriteHeader(http.StatusAccepted) + case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"): + repository := strings.TrimPrefix(r.URL.Path, "/upload/") + body, _ := io.ReadAll(r.Body) + digest := r.URL.Query().Get("digest") + if digest != digestOf(body) { + w.WriteHeader(http.StatusBadRequest) + return + } + m.blobs[repository+"@"+digest] = body + w.WriteHeader(http.StatusCreated) + case strings.Contains(path, "/blobs/"): + repository, digest, _ := strings.Cut(path, "/blobs/") + key := repository + "@" + digest + body, ok := m.blobs[key] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + switch r.Method { + case http.MethodHead: + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + delete(m.blobs, key) + w.WriteHeader(http.StatusAccepted) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + case strings.Contains(path, "/manifests/"): + repository, digest, _ := strings.Cut(path, "/manifests/") + key := repository + "@" + digest + switch r.Method { + case http.MethodHead: + if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + if digest != digestOf(body) { + w.WriteHeader(http.StatusBadRequest) + return + } + var named holderManifest + if err := json.Unmarshal(body, &named); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + for _, d := range append([]descriptor{named.Config}, named.Layers...) { + if _, ok := m.blobs[repository+"@"+d.Digest]; !ok { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest) + return + } + } + m.manifests[key] = body + w.WriteHeader(http.StatusCreated) + case http.MethodDelete: + if _, ok := m.manifests[key]; !ok { + w.WriteHeader(http.StatusNotFound) + return + } + delete(m.manifests, key) + w.WriteHeader(http.StatusAccepted) + } + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + return Store{Address: strings.TrimPrefix(server.URL, "http://")} +} + +// bare puts an archive in the store the way the builder did before holders: a blob, nothing more. +func (m *memRegistry) bare(repository string, body []byte) string { + m.mu.Lock() + defer m.mu.Unlock() + digest := digestOf(body) + m.blobs[repository+"@"+digest] = body + return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest +} + +func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) { + // The sweep must arrive at the very manifest the builder wrote, with nothing recorded between + // them. Same inputs, same bytes — and a different size is a different holder, so a holder can + // never be mistaken for one of a different blob. + digest := "sha256:" + strings.Repeat("a", 64) + one, first := Holder(digest, 42) + two, second := Holder(digest, 42) + if !bytes.Equal(one, two) || first != second { + t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two) + } + if _, other := Holder(digest, 43); other == first { + t.Fatal("a different size composed the same holder") + } + want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` + + `"config":{"mediaType":"application/vnd.oci.empty.v1+json",` + + `"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` + + `"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}` + if string(one) != want { + t.Fatalf("the holder is\n%s\nwant\n%s", one, want) + } + if digestOf(emptyConfig) != emptyDigest { + t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest) + } +} + +func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) { + // The archives published before this have no holder. Hold, run over every kept archive on + // every sweep, writes one the first time and nothing after. + m := &memRegistry{} + store := m.serve(t) + ctx := context.Background() + body := []byte("a theme") + reference := m.bare("shell/config", body) + + if held, err := store.Held(ctx, reference); err != nil || held { + t.Fatalf("a bare blob reads as held=%v (%v)", held, err) + } + wrote, err := store.Hold(ctx, reference) + if err != nil { + t.Fatal(err) + } + if !wrote { + t.Fatal("holding a bare archive wrote nothing") + } + _, holder := Holder(digestOf(body), int64(len(body))) + if _, ok := m.manifests["shell/config@"+holder]; !ok { + t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder) + } + if held, err := store.Held(ctx, reference); err != nil || !held { + t.Fatalf("after holding, held=%v (%v)", held, err) + } + + writes := len(m.writes) + wrote, err = store.Hold(ctx, reference) + if err != nil { + t.Fatal(err) + } + if wrote || len(m.writes) != writes { + t.Fatalf("holding again wrote %v", m.writes[writes:]) + } +} + +func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) { + m := &memRegistry{} + store := m.serve(t) + ctx := context.Background() + + // An image is its own manifest: nothing is asked. + wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64)) + if err != nil || wrote || len(m.writes) != 0 { + t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes) + } + // An archive the store does not have is a fact to report, not something to invent a holder for. + _, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64)) + if !errors.Is(err, Gone) { + t.Fatalf("holding a missing archive answered %v, want Gone", err) + } + // And a reference that is not the mesh's is refused as such. + if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) { + t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err) + } +} + +func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) { + // A holder left behind would keep the bytes through every collection while the record said + // collected; the link deleted first and the holder failing after would be that exactly. + m := &memRegistry{} + store := m.serve(t) + ctx := context.Background() + body := []byte("an old theme") + reference := m.bare("shell/config", body) + if _, err := store.Hold(ctx, reference); err != nil { + t.Fatal(err) + } + m.writes = nil + + if err := store.LetGo(ctx, reference); err != nil { + t.Fatal(err) + } + _, holder := Holder(digestOf(body), int64(len(body))) + want := []string{ + "DELETE /v2/shell/config/manifests/" + holder, + "DELETE /v2/shell/config/blobs/" + digestOf(body), + } + if strings.Join(m.writes, "\n") != strings.Join(want, "\n") { + t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n")) + } + if len(m.manifests) != 0 { + t.Fatalf("a holder survived: %v", m.manifests) + } + // Asked again, the archive is already gone, which is the outcome wanted. + if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) { + t.Fatalf("letting go twice answered %v, want Gone", err) + } +} + +func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) { + // An archive published before holders and let go of before any sweep held it: the holder's + // delete answers 404, which is the outcome wanted, and the blob still goes. + m := &memRegistry{} + store := m.serve(t) + reference := m.bare("shell/config", []byte("never held")) + if err := store.LetGo(context.Background(), reference); err != nil { + t.Fatal(err) + } + if len(m.blobs) != 0 { + t.Fatalf("the blob survived: %v", m.blobs) + } +} diff --git a/modules/artifact-store-tools/internal/registry/platforms_test.go b/modules/artifact-store-tools/internal/registry/platforms_test.go new file mode 100644 index 00000000..a335fb2c --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/platforms_test.go @@ -0,0 +1,176 @@ +package registry + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// An index is let go of with the platform manifests it names, except those a kept index of the same +// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in +// its own right, and the store's collector keeps every manifest a repository holds — so an index let go +// of alone frees nothing of the images it names. + +func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) } + +// indexStore holds indexes and images in one repository, answers GETs with their documents, and records +// every delete. +type indexStore struct { + mu sync.Mutex + indexes map[string][]string // digest → the platform manifests it names + images map[string]bool + deleted []string +} + +func (s *indexStore) serve(t *testing.T) Store { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + switch r.Method { + case http.MethodGet: + if children, ok := s.indexes[digest]; ok { + var named []string + for _, c := range children { + named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`) + } + w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") + _, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + + strings.Join(named, ",") + `]}`)) + return + } + if s.images[digest] { + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + _, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`)) + return + } + w.WriteHeader(http.StatusNotFound) + case http.MethodHead: + if _, ok := s.indexes[digest]; ok || s.images[digest] { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + case http.MethodDelete: + if _, ok := s.indexes[digest]; !ok && !s.images[digest] { + w.WriteHeader(http.StatusNotFound) + return + } + s.deleted = append(s.deleted, digest) + delete(s.indexes, digest) + delete(s.images, digest) + w.WriteHeader(http.StatusAccepted) + default: + w.WriteHeader(http.StatusBadRequest) + } + })) + t.Cleanup(server.Close) + return Store{Address: strings.TrimPrefix(server.URL, "http://")} +} + +func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) { + // Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13. + s := &indexStore{ + indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}}, + images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true}, + } + store := s.serve(t) + repository := "upstream/docker.io/library/golang" + old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1) + kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2) + spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}) + if err != nil { + t.Fatal(err) + } + store.Spare = spare + + if err := store.LetGo(context.Background(), old); err != nil { + t.Fatal(err) + } + // Its own platform first, the index last; the platform the kept index names is left in place. + if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) { + t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted) + } + if !s.images[digestN(12)] || !s.images[digestN(13)] { + t.Fatal("a platform a kept index names was let go of") + } +} + +func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) { + s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}} + store := s.serve(t) + // No Spare: as before ADR 0257, the index alone. + if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil { + t.Fatal(err) + } + if !slices.Equal(s.deleted, []string{digestN(1)}) { + t.Fatalf("without a Spare, deleted %v", s.deleted) + } + // An image is one delete, with a Spare or without. + s.deleted = nil + spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)}) + if err != nil { + t.Fatal(err) + } + store.Spare = spare + if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil { + t.Fatal(err) + } + if !slices.Equal(s.deleted, []string{digestN(5)}) { + t.Fatalf("an image was let go of as %v", s.deleted) + } + // An index the store no longer holds is Gone, and nothing is deleted on its account. + s.deleted = nil + err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9)) + if !errors.Is(err, Gone) || len(s.deleted) != 0 { + t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted) + } +} + +func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) { + s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}} + store := s.serve(t) + store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") } + err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)) + if err == nil || !strings.Contains(err.Error(), "was kept") { + t.Fatalf("a spare that could not be read: %v", err) + } + if len(s.deleted) != 0 { + t.Fatalf("deleted %v without knowing what a kept index names", s.deleted) + } +} + +// The spare list is read whole before the sweep: a kept index that cannot be read is an error before +// anything is deleted, and a repository not read before keeps its index. +func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(server.Close) + store := Store{Address: strings.TrimPrefix(server.URL, "http://")} + kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2) + old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1) + if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil { + t.Fatal("a kept index the store would not answer for was taken as naming nothing") + } + + s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}} + good := s.serve(t) + spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)}) + if err != nil { + t.Fatal(err) + } + good.Spare = spare + if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 { + t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted) + } +} diff --git a/modules/artifact-store-tools/internal/registry/store.go b/modules/artifact-store-tools/internal/registry/store.go new file mode 100644 index 00000000..8a1fe931 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/store.go @@ -0,0 +1,325 @@ +// Package registry speaks to the mesh's artifact store over its own door. +// +// Only what the mesh needs that nothing else does: letting go of something it put there +// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's +// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading +// is every machine's, through its runtime. This is the one operation that belongs to the thing +// holding the records, because it is the only one that is a decision rather than a transfer. +// +// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets +// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease. +package registry + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Store is the artifact store at an address, as this machine reaches it. +type Store struct { + // Address is `host:port` — the store as the caller reaches it now, composed and never + // recorded (novox/hq 04-ISSUES/102). + Address string + // HTTP is the client used; nil is a client with a modest timeout. + HTTP *http.Client + // Spare says, for one repository, which manifests a kept index there names: the platform + // manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every + // platform manifest — an index is then let go of alone, as it was before. + Spare func(ctx context.Context, repository string) (map[string]bool, error) +} + +// Gone is the answer when the store does not hold it: the outcome wanted, already true. +var Gone = errors.New("the store does not hold it") + +// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing +// the store holds by digest. +// +// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite +// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an +// error" is how a cautious loop became one that did nothing while reporting the right number. +var ErrNotOurs = errors.New("not a reference into the mesh's artifact store") + +// LetGo asks the store to drop one artifact the mesh recorded making. +// +// Takes a reference as the mesh records it — `artifact-store:///@sha256:…` for +// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses, +// and composes the address here at the moment of use. +// +// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq +// issue 253); an image in one. +// +// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep +// wants the artifact absent, and it is. It is distinguished from success only so a caller can say +// which of the two happened. +func (s Store) LetGo(ctx context.Context, reference string) error { + // **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its + // own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…` + // as the mesh's too — it cannot tell one registry host from another — so normalising belongs + // where the provenance is known, which is the sweep reading its own build records, not here + // where the only job is to refuse anything that is not plainly ours. + path, kept := catalogue.InArtifactStore(reference) + if !kept { + // Nothing the mesh put in its own store. Refused rather than attempted: composing a + // delete for a reference of unknown shape is how a sweep reaches something that is not + // the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on. + return fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + if s.Address == "" { + return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return err + } + if kind == "blobs" { + // **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left + // naming the blob would keep its bytes through every collection while the record said + // collected. Gone here means the store has no such blob, so there is nothing to let go. + if err := s.letGoOfHolder(ctx, repository, digest); err != nil { + return err + } + } + if kind == "manifests" && s.Spare != nil { + // **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put + // under its own digest, so each is a manifest of the repository in its own right, and the + // store's collector keeps every manifest a repository holds: an index let go of alone frees + // no byte of the images it names. Before, not after, so a sweep cut short leaves the index + // to be asked about again, and with it the list of what is still to go. + if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil { + return err + } + } + return s.remove(ctx, s.url(repository, kind, digest), reference) +} + +// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same +// repository also names. A manifest that is not an index names none. +func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error { + children, err := s.Platforms(ctx, repository, digest) + if err != nil || len(children) == 0 { + // Gone: there is no index to read, and the delete that follows says so. + if errors.Is(err, Gone) { + return nil + } + return err + } + spared, err := s.Spare(ctx, repository) + if err != nil { + return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w", + repository, reference, err) + } + for _, child := range children { + if spared[child] { + continue + } + if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) { + return err + } + } + return nil +} + +// anyManifest is every manifest the store may hold, indexes included, so it answers with the +// document as it is. +var anyManifest = []string{ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +} + +// Platforms is the manifests an index names, by digest; none for a manifest that is not an index. +// Gone when the store does not hold it. +func (s Store) Platforms(ctx context.Context, repository, digest string) ([]string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "manifests", digest), nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", strings.Join(anyManifest, ", ")) + response, err := s.client().Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, Gone + default: + return nil, fmt.Errorf("the artifact store answered %s when asked for %s@%s", response.Status, repository, digest) + } + var document struct { + Manifests []struct { + Digest string `json:"digest"` + } `json:"manifests"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil { + return nil, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err) + } + var out []string + for _, m := range document.Manifests { + if strings.HasPrefix(m.Digest, "sha256:") { + out = append(out, m.Digest) + } + } + return out, nil +} + +// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an +// image reference of `touching` is in, each kept manifest there and every platform manifest a kept +// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not +// read here is answered with an error, so an index in it is kept. A kept index the store no longer +// holds names nothing. +func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) { + imageIn := func(reference string) (repository, digest string, ok bool) { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return "", "", false + } + repository, kind, digest, err := split(path) + if err != nil || kind != "manifests" { + return "", "", false + } + return repository, digest, true + } + read := map[string]map[string]bool{} + for _, reference := range touching { + if repository, _, ok := imageIn(reference); ok { + read[repository] = map[string]bool{} + } + } + for _, reference := range kept { + repository, digest, ok := imageIn(reference) + spared, touched := read[repository] + if !ok || !touched { + continue + } + spared[digest] = true + children, err := s.Platforms(ctx, repository, digest) + if errors.Is(err, Gone) { + continue + } + if err != nil { + return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err) + } + for _, c := range children { + spared[c] = true + } + } + return func(_ context.Context, repository string) (map[string]bool, error) { + spared, done := read[repository] + if !done { + return nil, fmt.Errorf("%s was not read before the sweep began", repository) + } + return spared, nil + }, nil +} + +// remove asks the store to delete what is at url. Gone when it has no such thing. +func (s Store) remove(ctx context.Context, url, what string) error { + request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) + if err != nil { + return err + } + response, err := s.client().Do(request) + if err != nil { + return err + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusAccepted, http.StatusOK, http.StatusNoContent: + return nil + case http.StatusNotFound: + return Gone + case http.StatusMethodNotAllowed: + // The registry was started without deletion enabled. Said plainly, because the remedy is + // a setting on the store's module and not anything about this artifact. + return fmt.Errorf( + "the artifact store refuses deletion: its server was started without it enabled "+ + "(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+ + "module is applied again (novox/hq ADR 0189). Asking about %s", what) + default: + return fmt.Errorf("the artifact store answered %s for %s", response.Status, what) + } +} + +// split reads a recorded path into the repository, which endpoint names the thing, and the digest. +// +// Two shapes, which are the two the mesh records: `@sha256:` is a manifest, and +// `/blobs/sha256:` is a blob. +func split(path string) (repository, kind, digest string, err error) { + if before, after, ok := strings.Cut(path, "@sha256:"); ok { + return before, "manifests", "sha256:" + after, nil + } + if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok { + return before, "blobs", "sha256:" + after, nil + } + return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path) +} + +// IsIndex is whether a recorded reference is an index the store holds: one that names manifests. A +// blob, or a manifest that names none, is not. Gone when the store does not hold it. +func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return false, fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + repository, kind, digest, err := split(path) + if err != nil || kind != "manifests" { + return false, err + } + children, err := s.Platforms(ctx, repository, digest) + return len(children) > 0, err +} + +// HoldsManifest is whether the store holds the manifest a recorded image reference names. +func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return false, fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return false, err + } + if kind != "manifests" { + return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference) + } + return s.has(ctx, s.url(repository, kind, digest), anyManifest...) +} + +// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image. +// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that +// are not there). +func (s Store) Holds(ctx context.Context, reference string) error { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return err + } + if s.Address == "" { + return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) + } + if kind == "blobs" { + _, err := s.blobSize(ctx, repository, digest) + return err + } + held, err := s.has(ctx, s.url(repository, kind, digest), anyManifest...) + if err != nil { + return err + } + if !held { + return Gone + } + return nil +} diff --git a/modules/artifact-store-tools/internal/registry/store_test.go b/modules/artifact-store-tools/internal/registry/store_test.go new file mode 100644 index 00000000..334b95a5 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/store_test.go @@ -0,0 +1,134 @@ +package registry + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108). +// +// A fake store records what it was asked to delete, so what is asserted is the mesh's decision +// and the shape of the request — not the registry's behaviour, which is the registry's to test. + +func fakeStore(t *testing.T, answer int) (Store, *[]string) { + t.Helper() + var asked []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") { + // An archive's size, asked so its holder can be named (novox/hq issue 253). A store + // that does not have the thing does not have its blob either. + if answer == http.StatusNotFound { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Length", "7") + w.WriteHeader(http.StatusOK) + return + } + if r.Method != http.MethodDelete { + t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path) + } + asked = append(asked, r.URL.Path) + w.WriteHeader(answer) + })) + t.Cleanup(server.Close) + return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked +} + +func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) { + // The two shapes the mesh records: a manifest by digest, and a blob by digest. They are + // different endpoints, and asking at the wrong one answers 404 — which this would then + // record as collected, leaving the bytes on disk for ever while the record says otherwise. + store, asked := fakeStore(t, http.StatusAccepted) + ctx := context.Background() + + image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123" + archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456" + if err := store.LetGo(ctx, image); err != nil { + t.Fatal(err) + } + if err := store.LetGo(ctx, archive); err != nil { + t.Fatal(err) + } + // The archive's holder goes first, then the archive (novox/hq issue 253). + _, holder := Holder("sha256:def456", 7) + want := []string{ + "/v2/web/app/manifests/sha256:abc123", + "/v2/web/config/manifests/" + holder, + "/v2/web/config/blobs/sha256:def456", + } + if strings.Join(*asked, " ") != strings.Join(want, " ") { + t.Fatalf("the store was asked %v; want %v", *asked, want) + } +} + +func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) { + // The outcome wanted, already true. Told apart from success only so the sweep can say which + // happened; both are recorded, because retrying for ever is the thing to avoid. + store, _ := fakeStore(t, http.StatusNotFound) + err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123") + if !errors.Is(err, Gone) { + t.Fatalf("a store that does not hold it answered %v, want Gone", err) + } +} + +func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) { + // The registry answers 405 when it was started without deletion enabled. The remedy is a + // setting on the store's module, and saying "405" would send somebody to the wrong place. + store, _ := fakeStore(t, http.StatusMethodNotAllowed) + err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123") + if err == nil { + t.Fatal("a store that refuses deletion was read as success") + } + if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") { + t.Fatalf("the refusal does not name the remedy: %v", err) + } +} + +func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) { + // The whole safety of the sweep is that it names only what the mesh recorded putting there. + // A reference of another shape — a vendor's image, a package version — is refused rather + // than composed into a delete somewhere that is not the mesh's store. + store, asked := fakeStore(t, http.StatusAccepted) + for _, reference := range []string{ + "docker.io/library/registry@sha256:abc123", + "registry@sha256:abc123", + "1.4.2", + } { + if err := store.LetGo(context.Background(), reference); err == nil { + t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference) + } + } + if len(*asked) != 0 { + t.Fatalf("the store was asked about %v", *asked) + } +} + +// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the +// record and not about the store (novox/hq issue 226). +// +// The sweep skips one and abandons itself for the other, so they cannot be the same error. The +// first live run met a reference recorded with the store's old address, read the refusal as "the +// store refuses everything", and collected none of the 1681 it had found. +func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) { + store, asked := fakeStore(t, http.StatusAccepted) + for _, reference := range []string{ + "docker.io/library/registry@sha256:abc123", + "127.0.0.1:5100/mesh-tools/build@sha256:abc123", + "1.4.2", + } { + err := store.LetGo(context.Background(), reference) + if !errors.Is(err, ErrNotOurs) { + t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err) + } + } + if len(*asked) != 0 { + t.Fatalf("the store was asked about %v", *asked) + } +} diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index 208329cd..f1ef7a43 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -1,26 +1,52 @@ { "module": "artifact-store-tools", "version": "1", + "claims": [ + { + "name": "mesh-artifact-sweep", + "scope": "mesh", + "serves": [ + "artifacts", + "held", + "collect", + "mirrors", + "images" + ] + } + ], + "requires": [ + "artifact-store" + ], "invokes": [ - "seat:mesh-controller.artifacts", - "seat:mesh-controller.collect", + "seat:mesh-hand-acts.hand-act", "mesh-catalog.catalog_modules", "mesh-catalog.catalog_module" ], + "reads": [ + "mesh-delivery-assigner.wants", + "mesh-delivery-builder.builds", + "mesh-delivery-builder.build-log", + "mesh-delivery-indexer.manifests", + "node-engine.have", + "mesh-controller.lease" + ], + "state": [ + "findings" + ], "tools": [ "artifact_store_repositories", "artifact_store_usage", "artifact_store_references", "artifact_store_collect" ], - "config": { - "older-store-names": { - "type": "string", - "description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).", - "pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*", - "default": "none" - } - }, + "config": { + "older-store-names": { + "type": "string", + "description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).", + "pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*", + "default": "none" + } + }, "resources": [ { "id": "state", @@ -36,6 +62,14 @@ "mode": "0600", "trusted": false, "template": "files/store-names" + }, + { + "id": "store-address", + "type": "file", + "path": "${dir:state}/store-address", + "mode": "0600", + "trusted": false, + "template": "files/store-address" } ], "build": { @@ -52,7 +86,8 @@ ], "env": { "MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry", - "MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names" + "MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names", + "MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address" } }, { diff --git a/modules/buildah/module.json b/modules/buildah/module.json index 70ddd0a4..bdfbe96b 100644 --- a/modules/buildah/module.json +++ b/modules/buildah/module.json @@ -8,7 +8,10 @@ "provides": [ { "name": "rootless-image-build", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "resources": [ diff --git a/modules/disk-load/.gitignore b/modules/disk-load/.gitignore new file mode 100644 index 00000000..96984eae --- /dev/null +++ b/modules/disk-load/.gitignore @@ -0,0 +1 @@ +disk-load diff --git a/modules/disk-load/cmd/disk-load/keeper.go b/modules/disk-load/cmd/disk-load/keeper.go new file mode 100644 index 00000000..388ef136 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/keeper.go @@ -0,0 +1,248 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" + "os" + "os/exec" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Store is the state `readings` as the keeper uses it. +type Store interface { + Get(key string) (json.RawMessage, error) + Put(key string, value any) error + Keys() ([]string, error) +} + +// busStore is the state through the runtime. +type busStore struct{} + +func (busStore) Get(key string) (json.RawMessage, error) { + e, err := stdio.State(StateReadings).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} +func (busStore) Put(key string, v any) error { + _, err := stdio.State(StateReadings).Put(key, v) + return err +} +func (busStore) Keys() ([]string, error) { return stdio.State(StateReadings).Keys() } + +// Keeper measures the items of its node and keeps the readings. +type Keeper struct { + Node string + // Items is every data item the modules placed on this node declare, composed from the bus (frombus.DataItems). + Items func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) + Measurer Measurer + Store Store + Say func(string, ...any) + + mu sync.Mutex // one round at a time +} + +// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item +// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round +// reads counters, and a walk runs at most once a day (ADR 0233 §22). +func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) { + k.mu.Lock() + defer k.mu.Unlock() + items, err := k.Items(ctx) + if err != nil { + return 0, 0, err + } + modules := make([]string, 0, len(items)) + for m := range items { + modules = append(modules, m) + } + sort.Strings(modules) + var failed []string + for _, module := range modules { + for _, it := range items[module] { + if it.Class == ClassCache { + continue + } + w, err := k.measureOne(ctx, it, walks) + if err != nil { + failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err)) + continue + } + if w { + written++ + } else { + kept++ + } + } + } + retired, err := k.retireGone(items) + written += retired + if err != nil { + failed = append(failed, err.Error()) + } + if len(failed) > 0 { + return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; ")) + } + return written, kept, nil +} + +// retireGone marks retired every reading of this node whose item the node no longer declares — its module was unassigned +// there, or is no longer pulled in — with when and why, and keeps it: the record of what the module left behind, until a +// person deletes it (ADR 0233). It is cleared when the item is declared again (measureOne). +func (k *Keeper) retireGone(declared map[string][]catalogue.PlacedDataItem) (int, error) { + keys, err := k.Store.Keys() + if err != nil { + return 0, err + } + declaredNow := map[string]bool{} + for module, its := range declared { + for _, it := range its { + declaredNow[health.ReadingKey(k.Node, module, it.ID)] = true + } + } + now := k.Measurer.Now().UTC() + n := 0 + for _, key := range keys { + if !strings.HasPrefix(key, k.Node+".") || declaredNow[key] { + continue + } + raw, err := k.Store.Get(key) + if err != nil || len(raw) == 0 { + continue + } + r, err := health.DecodeReading(raw) + if err != nil || r.Node != k.Node || !r.RetiredAt.IsZero() { + continue + } + r.RetiredAt = now + r.RetiredWhy = fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", k.Node) + if err := k.Store.Put(key, r); err != nil { + return n, err + } + n++ + } + return n, nil +} + +func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, error) { + key := health.ReadingKey(k.Node, it.Module, it.ID) + var prev *health.Reading + raw, err := k.Store.Get(key) + if err != nil { + return false, err + } + if len(raw) > 0 { + if p, err := health.DecodeReading(raw); err == nil { + prev = &p + } + } + now := k.Measurer.Now() + if prev != nil && prev.RetiredAt.IsZero() && it.Measure == catalogue.MeasureWalk && !walks && prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery { + return false, nil + } + m := k.Measurer.Measure(ctx, it) + next, changed := health.NextReading(prev, k.Node, it, m, now) + if prev != nil && !prev.RetiredAt.IsZero() { + // Declared again: it is no longer retired. + next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true + } + if !changed { + return false, nil + } + return true, k.Store.Put(key, next) +} + +// MeasureNow measures one item now, walk or not, and writes its reading when it changed. +func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) { + k.mu.Lock() + defer k.mu.Unlock() + items, err := k.Items(ctx) + if err != nil { + return health.Reading{}, false, err + } + for _, it := range items[module] { + if it.ID != item { + continue + } + if it.Class == ClassCache { + return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item) + } + w, err := k.measureOne(ctx, it, true) + if err != nil { + return health.Reading{}, false, err + } + raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item)) + if err != nil { + return health.Reading{}, w, err + } + r, err := health.DecodeReading(raw) + return r, w, err + } + return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item) +} + +// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the +// hour after the night; ctx ends it. +func (k *Keeper) Loop(ctx context.Context, every time.Duration, first time.Duration) { + select { + case <-ctx.Done(): + return + case <-time.After(first): + } + for { + w, kept, err := k.Round(ctx, false) + if err != nil { + k.Say("measuring the data declared here: %v", err) + } else { + k.Say("measured the data declared here: %d readings changed, %d unchanged", w, kept) + } + select { + case <-ctx.Done(): + return + case <-time.After(every): + } + } +} + +// RootRunner runs a command as root, through sudo without a prompt when this process is not root. A walk is bounded +// by WalkFor itself; the runner's own bound is a little longer. +func RootRunner(ctx context.Context, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, WalkFor+time.Minute) + defer cancel() + program, argv := name, args + if os.Getuid() != 0 { + program, argv = "sudo", append([]string{"-n", name}, args...) + } + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, program, argv...) + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + said := strings.TrimSpace(stderr.String()) + if said == "" { + said = strings.TrimSpace(stdout.String()) + } + if program == "sudo" && strings.HasPrefix(said, "sudo:") { + return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said) + } + lines := strings.Split(said, "\n") + if len(lines) > 3 { + lines = lines[len(lines)-3:] + } + if said == "" { + return "", fmt.Errorf("%s: %w", name, err) + } + return "", errors.New(strings.Join(lines, " / ")) + } + return stdout.String(), nil +} diff --git a/modules/disk-load/cmd/disk-load/main.go b/modules/disk-load/cmd/disk-load/main.go index 92fd2ef4..2eeff0b8 100644 --- a/modules/disk-load/cmd/disk-load/main.go +++ b/modules/disk-load/cmd/disk-load/main.go @@ -1,20 +1,61 @@ // The disk-load module's Go bundle (novox/hq ADR 0188, ADR 0193, issue 312): a process the node's // runtime launches and speaks MCP over stdio to. It reads the machine's storage load from /proc and // /sys — iowait, pressure stall, each disk's busy share, throughput, IOPS, wait and queue depth, the -// filesystems' space and the processes doing the I/O — and changes nothing. +// filesystems' space and the processes doing the I/O — and changes nothing of the machine. +// +// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's +// seven-day peak as its own state `readings` (ADR 0571 clause 3), composed from the bus (no file). package main import ( + "context" "fmt" "os" + "strings" + "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/compose/fromruntime" ) +func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) } + func main() { + if err := stdio.Keeps(announce.State{Name: StateReadings}); err != nil { + logf("%v", err) + os.Exit(1) + } + node := os.Getenv("MESH_NODE") + k := &Keeper{Node: node, Store: busStore{}, Say: logf, Measurer: Measurer{Run: RootRunner, Now: time.Now}, + Items: func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) { + return placedItems(ctx, node) + }} + if node != "" { + every := time.Hour + if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute { + every = d + } + go k.Loop(context.Background(), every, 3*time.Minute) + } + tools := append(Tools(Here()), ReadingTools(k)...) // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load. - if err := stdio.Serve("", Tools(Here())); err != nil { + if err := stdio.Serve("", tools); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } + +// placedItems is every data item the modules placed on node declare, composed from the bus as the node composes itself. +func placedItems(ctx context.Context, node string) (map[string][]catalogue.PlacedDataItem, error) { + items, unplaced, err := frombus.DataItems(ctx, fromruntime.Reader(), frombus.Mesh{}, frombus.Scope{Node: node}) + if err != nil { + return nil, err + } + if len(unplaced) > 0 { + logf("data items left out as not placeable: %s", strings.Join(unplaced, "; ")) + } + return items, nil +} diff --git a/modules/disk-load/cmd/disk-load/measure.go b/modules/disk-load/cmd/disk-load/measure.go new file mode 100644 index 00000000..2973080b --- /dev/null +++ b/modules/disk-load/cmd/disk-load/measure.go @@ -0,0 +1,215 @@ +package main + +// How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the +// readings now (ADR 0571 clause 3). Never an unbounded walk of something large. + +import ( + "context" + "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" + "regexp" + "strconv" + "strings" + "time" +) + +// The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so. +var ( + WalkFor = 10 * time.Minute + WalkFiles = 2_000_000 + WalkEvery = 23 * time.Hour +) + +// walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files. +// One line out: " ", then the status of find. +func walkCommand(path string) string { + return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ + "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ + "echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles) +} + +// topCommand is the newest change among a path and its top-level entries, and how many there are. +func topCommand(path string) string { + return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + + "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" +} + +func epochTime(s string) *time.Time { + epoch, err := strconv.ParseInt(s, 10, 64) + if err != nil || epoch <= 0 { + return nil + } + t := time.Unix(epoch, 0).UTC() + return &t +} + +// Measurer measures items through a runner that has the root the walks need. +type Measurer struct { + Run Runner + Now func() time.Time +} + +func (b Measurer) exists(ctx context.Context, path string) bool { + _, err := b.Run(ctx, "test", "-e", path) + return err == nil +} + +// Measure is one item, measured now, by its own method. +func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured { + at := b.Now().UTC() + m := health.Measured{MeasuredAt: &at} + if !b.exists(ctx, it.Path) { + m.Error = it.Path + " does not exist" + zero := int64(0) + m.SizeBytes, m.Precision = &zero, "exact" + return m + } + m.Redundancy = b.redundancyOf(ctx, it.Path) + switch it.Measure { + case catalogue.MeasureDataset, catalogue.MeasureShallow: + out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) + if f := strings.Fields(out); err == nil && len(f) == 2 { + m.LastWrite = epochTime(f[0]) + } else if err != nil { + m.Error = err.Error() + } + m.Precision = "no size: the newest change among its top-level entries only" + if it.Measure == catalogue.MeasureShallow { + return m + } + out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) + f := strings.Fields(out) + if err != nil || len(f) != 2 { + m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" + return m + } + used, err := strconv.ParseInt(f[1], 10, 64) + if err != nil { + return m + } + m.SizeBytes = &used + m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" + return m + } + out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) + if err != nil { + m.Error = err.Error() + return m + } + lines := strings.Split(strings.TrimSpace(out), "\n") + f := strings.Fields(lines[0]) + if len(f) != 3 || len(lines) < 2 { + m.Error = "the measurement said " + strings.TrimSpace(out) + return m + } + size, err1 := strconv.ParseInt(f[0], 10, 64) + files, err2 := strconv.Atoi(f[2]) + if err1 != nil || err2 != nil { + m.Error = "the measurement said " + strings.TrimSpace(out) + return m + } + m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) + m.Precision = "exact" + if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles { + m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ + "and this item wants measure: dataset or shallow", WalkFor, WalkFiles) + } + return m +} + +// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs +// filesystem's error counters. Nil for storage with no redundancy it knows how to read. +func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy { + out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) + if err != nil { + return nil + } + f := strings.Fields(out) + if len(f) < 2 { + return nil + } + source, fstype := f[0], f[1] + yes, no := true, false + switch { + case fstype == "zfs": + pool, _, _ := strings.Cut(source, "/") + r := &health.Redundancy{Kind: "zfs", Where: pool} + health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) + if err != nil { + r.Said = "zpool list: " + err.Error() + return r + } + status, err := b.Run(ctx, "zpool", "status", "-x", pool) + if err != nil { + r.Said = "zpool status: " + err.Error() + return r + } + health, status = strings.TrimSpace(health), strings.TrimSpace(status) + r.Said = "health " + health + "; " + firstLineOf(status) + if health == "ONLINE" && strings.Contains(status, "is healthy") { + r.Healthy = &yes + } else { + r.Healthy = &no + r.Said = "health " + health + "; " + oneLineOf(status) + } + return r + case strings.HasPrefix(source, "/dev/md"): + device := strings.TrimPrefix(source, "/dev/") + r := &health.Redundancy{Kind: "md", Where: device} + mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") + if err != nil { + r.Said = err.Error() + return r + } + healthy, said, found := mdHealth(mdstat, device) + if !found { + r.Said = device + " is not in /proc/mdstat" + return r + } + r.Said = said + if healthy { + r.Healthy = &yes + } else { + r.Healthy = &no + } + return r + case fstype == "btrfs": + r := &health.Redundancy{Kind: "btrfs", Where: source} + stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) + if err != nil { + r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) + return r + } + r.Healthy, r.Said = &yes, firstLineOf(stats) + return r + } + return nil +} + +var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) + +// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering. +func mdHealth(mdstat, device string) (bool, string, bool) { + lines := strings.Split(mdstat, "\n") + for i, l := range lines { + if !strings.HasPrefix(l, device+" ") { + continue + } + block := l + for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { + block += " " + strings.TrimSpace(lines[j]) + } + members := mdMembers.FindStringSubmatch(block) + healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") + return healthy, oneLineOf(block), true + } + return false, "", false +} + +func firstLineOf(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} + +func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } diff --git a/modules/disk-load/cmd/disk-load/readings.go b/modules/disk-load/cmd/disk-load/readings.go new file mode 100644 index 00000000..3cad1734 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/readings.go @@ -0,0 +1,12 @@ +package main + +// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571 +// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state. + +import "git.novox.be/novox/mesh-sdk/go/health" + +// StateReadings is the state disk-load keeps. +const StateReadings = health.ReadingsState + +// ClassCache is the class of an item that is listed and never measured. +const ClassCache = "cache" diff --git a/modules/disk-load/cmd/disk-load/readings_test.go b/modules/disk-load/cmd/disk-load/readings_test.go new file mode 100644 index 00000000..d934ad27 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/readings_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" +) + +type memStore struct{ m map[string]json.RawMessage } + +func (s *memStore) Get(k string) (json.RawMessage, error) { return s.m[k], nil } +func (s *memStore) Put(k string, v any) error { + raw, _ := json.Marshal(v) + s.m[k] = raw + return nil +} +func (s *memStore) Keys() ([]string, error) { + var out []string + for k := range s.m { + out = append(out, k) + } + return out, nil +} + +// A round measures what the data file declares but caches, keeps the readings, writes nothing the second time, and a +// walk is not repeated within a day unless asked. +func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) { + text := "# pg\nitem data valuable /srv/pg /srv/pg backup walk\nitem tmp cache /srv/tmp - none\n# bad-line-module\nitem logs rebuildable /srv/logs - none shallow\n" + placed, err := catalogue.ParseDataItems(text) + if err != nil { + t.Fatal(err) + } + walks := 0 + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "test": + return "", nil + case name == "findmnt": + return "", os.ErrNotExist + case name == "bash" && strings.Contains(args[1], "-maxdepth 1"): + return "1760000000 2\n", nil + case name == "bash": + walks++ + return "4096 1760000000 3\n0\n", nil + } + return "", os.ErrNotExist + } + now := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + store := &memStore{m: map[string]json.RawMessage{}} + k := &Keeper{Node: "ace", Items: func(context.Context) (map[string][]catalogue.PlacedDataItem, error) { return placed, nil }, Store: store, Say: func(string, ...any) {}, + Measurer: Measurer{Run: run, Now: func() time.Time { return now }}} + w, kept, err := k.Round(context.Background(), false) + if err != nil || w != 2 || kept != 0 || len(store.m) != 2 { + t.Fatalf("first round: %d written %d kept %v, keys %d", w, kept, err, len(store.m)) + } + now = now.Add(time.Hour) + if w, kept, _ := k.Round(context.Background(), false); w != 0 || kept != 2 || walks != 1 { + t.Fatalf("second round: %d written, %d kept, %d walks; wanted none written and the walk not repeated", w, kept, walks) + } + if _, _, err := k.MeasureNow(context.Background(), "pg", "data"); err != nil || walks != 2 { + t.Fatalf("measure now: %v, %d walks", err, walks) + } + // An item no longer declared is kept, retired with when and why; declared again, it is not retired any more. + delete(placed, "pg") + if _, _, err := k.Round(context.Background(), false); err != nil { + t.Fatal(err) + } + gone, _ := health.DecodeReading(store.m["ace.pg.data"]) + if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() { + t.Fatalf("a reading of an item no longer declared: %+v", gone) + } + placed["pg"] = []catalogue.PlacedDataItem{{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureWalk}} + if _, _, err := k.Round(context.Background(), false); err != nil { + t.Fatal(err) + } + if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() { + t.Fatalf("a reading declared again is still retired: %+v", back) + } + if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil { + t.Fatal("a cache was measured") + } +} + +// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a member missing; and +// plain storage, which is no redundancy at all. A dataset is measured from the filesystem's counters and a shallow item +// from its top-level entries only; a walk that meets more than its bound stops and says so. +func TestTheStorageUnderAnItemAndTheLargeMethodsAreRead(t *testing.T) { + poolHealth, statusX, fs := "ONLINE\n", "pool 'storage' is healthy\n", "storage/media zfs\n" + run := func(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + switch { + case name == "test": + return "", nil + case name == "findmnt": + return fs, nil + case name == "zpool" && args[0] == "list": + return poolHealth, nil + case name == "zpool": + return statusX, nil + case name == "cat": + return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil + case name == "zfs" && args[len(args)-1] == "/storage/media/movies": + return "storage/media\t97067690722560\n", nil + case name == "zfs": + return "", os.ErrNotExist + case name == "bash" && strings.Contains(line, "-maxdepth 1"): + return "1759744800 12673\n", nil + case name == "bash": + return "123 1759700000 2000000\n141\n", nil + } + return "", nil + } + m := Measurer{Run: run, Now: time.Now} + ctx := context.Background() + got := m.Measure(ctx, catalogue.PlacedDataItem{Module: "media", ID: "movies", Path: "/storage/media/movies", Measure: catalogue.MeasureDataset}) + if got.SizeBytes == nil || *got.SizeBytes != 97067690722560 || !strings.HasPrefix(got.Precision, "dataset storage/media") || + got.LastWrite == nil || got.Redundancy == nil || got.Redundancy.Kind != "zfs" || got.Redundancy.Healthy == nil || !*got.Redundancy.Healthy { + t.Fatalf("the library from its dataset: %+v", got) + } + poolHealth, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" + if r := m.redundancyOf(ctx, "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { + t.Fatalf("a degraded pool: %+v", r) + } + fs = "/dev/md127 ext4\n" + if r := m.redundancyOf(ctx, "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { + t.Fatalf("an array missing a member: %+v", r) + } + fs = "/dev/nvme0n1p3 ext4\n" + if r := m.redundancyOf(ctx, "/var/lib/x"); r != nil { + t.Fatalf("plain storage read as redundant: %+v", r) + } + if s := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/mnt/plex/config", Measure: catalogue.MeasureShallow}); s.SizeBytes != nil || !strings.HasPrefix(s.Precision, "no size") { + t.Fatalf("a shallow item: %+v", s) + } + if w := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/srv/big", Measure: catalogue.MeasureWalk}); !strings.HasPrefix(w.Precision, "partial") { + t.Fatalf("a walk stopped at its bound: %+v", w) + } +} diff --git a/modules/disk-load/cmd/disk-load/readings_tools.go b/modules/disk-load/cmd/disk-load/readings_tools.go new file mode 100644 index 00000000..322fe275 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/readings_tools.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item +// measured now (a module's tools are its command line). +func ReadingTools(k *Keeper) []stdio.Tool { + return []stdio.Tool{ + { + Name: "disk_readings", + Description: "The data readings disk-load keeps for this machine's data items (ADR 0233, ADR 0571 clause 3): every key " + + "`..` with its size, newest write, measured-at, precision and seven-day peak, or one key with " + + "all it holds when `key` is given; and how full the state is: keys kept, their bytes, and the cap the bus puts on " + + "a state (none declared). Filter by `module`. (r)", + Input: map[string]any{ + "key": map[string]any{"type": "string", "description": "one key, .., with everything it holds"}, + "module": map[string]any{"type": "string", "description": "only this module's items"}, + }, + Run: func(args map[string]any) (any, error) { + if key := text(args, "key"); key != "" { + raw, err := k.Store.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no reading is kept at %q", key) + } + r, err := health.DecodeReading(raw) + return r, err + } + keys, err := k.Store.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + module := text(args, "module") + type line struct { + Key string `json:"key"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + PeakBytes *int64 `json:"peak_bytes,omitempty"` + Precision string `json:"precision,omitempty"` + Error string `json:"error,omitempty"` + Reading health.Reading `json:"-"` + } + var lines []line + total := 0 + for _, key := range keys { + raw, err := k.Store.Get(key) + if err != nil { + return nil, err + } + total += len(raw) + r, err := health.DecodeReading(raw) + if err != nil || (module != "" && r.Module != module) { + continue + } + lines = append(lines, line{Key: key, SizeBytes: r.SizeBytes, PeakBytes: r.Peak, Precision: r.Precision, Error: r.Error}) + } + return map[string]any{"readings": lines, "fill": map[string]any{"keys": len(keys), "bytes": total, + "cap": "none declared: the bus's own limit on a state applies"}}, nil + }, + }, + { + Name: "disk_measure", + Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " + + "changed. Reads the disks as root: a walk is bounded by time and files and says when it was partial.", + Input: map[string]any{ + "module": map[string]any{"type": "string", "description": "the module that declares the item"}, + "item": map[string]any{"type": "string", "description": "the item's id"}, + }, + Run: func(args map[string]any) (any, error) { + module, item := text(args, "module"), text(args, "item") + if module == "" || item == "" { + return nil, fmt.Errorf("module and item are required") + } + r, written, err := k.MeasureNow(context.Background(), module, item) + if err != nil { + return nil, err + } + return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item), + "note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil + }, + }, + } +} diff --git a/modules/disk-load/cmd/disk-load/tools_test.go b/modules/disk-load/cmd/disk-load/tools_test.go index 88147715..672e6e94 100644 --- a/modules/disk-load/cmd/disk-load/tools_test.go +++ b/modules/disk-load/cmd/disk-load/tools_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "os" + "regexp" "sort" "strings" "testing" @@ -35,9 +36,11 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T t.Fatal(err) } var served []string - for _, tl := range Tools(newFake(t).machine()) { + k := &Keeper{Store: &memStore{m: map[string]json.RawMessage{}}} + for _, tl := range append(Tools(newFake(t).machine()), ReadingTools(k)...) { served = append(served, tl.Name) - if !strings.HasSuffix(tl.Description, "(r)") { + // disk_measure writes the module's own state of readings and nothing else of the machine. + if !strings.HasSuffix(tl.Description, "(r)") && tl.Name != "disk_measure" { t.Errorf("%s only reads and does not say so", tl.Name) } } @@ -50,7 +53,7 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T t.Errorf("the module only reads, and declares %d resources", len(m.Resources)) } for _, banned := range []string{"g14", "novox", "ace", "shanks", "jochen", "/home/"} { - if strings.Contains(string(raw), banned) { + if regexp.MustCompile(`(^|[^a-z0-9-])` + regexp.QuoteMeta(banned) + `($|[^a-z0-9-])`).Match(raw) { t.Errorf("the manifest says %q", banned) } } diff --git a/modules/disk-load/go.mod b/modules/disk-load/go.mod index edf8078e..c3b85927 100644 --- a/modules/disk-load/go.mod +++ b/modules/disk-load/go.mod @@ -1,5 +1,5 @@ module diskload -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.6 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/disk-load/go.sum b/modules/disk-load/go.sum index 0dd60617..09ad064e 100644 --- a/modules/disk-load/go.sum +++ b/modules/disk-load/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/disk-load/module.json b/modules/disk-load/module.json index 9b48041d..4b4313ca 100644 --- a/modules/disk-load/module.json +++ b/modules/disk-load/module.json @@ -7,7 +7,9 @@ "disk_filesystems", "disk_folders", "disk_devices", - "cpu_top" + "cpu_top", + "disk_readings", + "disk_measure" ], "build": { "artifacts": [ @@ -23,5 +25,16 @@ ] } ] - } + }, + "state": [ + "readings" + ], + "reads": [ + "mesh-config.resolved", + "mesh-delivery-assigner.placement", + "mesh-delivery-builder.builds", + "mesh-delivery-indexer.manifests", + "mesh-delivery-indexer.modules", + "node-engine.have" + ] } diff --git a/modules/distribution/module.json b/modules/distribution/module.json index eb9f7c24..5037f429 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -4,7 +4,10 @@ "provides": [ { "name": "artifact-store", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "claims": [ diff --git a/modules/dnsmasq/go.mod b/modules/dnsmasq/go.mod index d439da63..e0d8d944 100644 --- a/modules/dnsmasq/go.mod +++ b/modules/dnsmasq/go.mod @@ -1,5 +1,5 @@ module dnsmasq -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/dnsmasq/go.sum b/modules/dnsmasq/go.sum index b474419e..f8bfbd2b 100644 --- a/modules/dnsmasq/go.sum +++ b/modules/dnsmasq/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 58cc0da5..7e832067 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -9,7 +9,10 @@ { "name": "wildcard-resolution", "scope": "mesh", - "identity": false + "identity": false, + "credential": { + "form": "none" + } } ], "requires": [ diff --git a/modules/docker/cmd/docker-tools/prune_images.go b/modules/docker/cmd/docker-tools/prune_images.go index 49f75436..e15f7fdf 100644 --- a/modules/docker/cmd/docker-tools/prune_images.go +++ b/modules/docker/cmd/docker-tools/prune_images.go @@ -9,7 +9,7 @@ package main // // - every image a container on this machine uses, in any state; // - every image this machine's declaration names — what the mesh would send it now and what it was -// last sent — as the controller's `images` verb answers. **No answer, no removal**: what the +// last sent — as the mesh-artifact-sweep seat's `images` verb answers. **No answer, no removal**: what the // declaration names is the one thing that cannot be guessed; // - every image younger than the floor (seven days, the weekly prune's week); // - in each line of images holding one of those, the newest image besides them: the previous @@ -43,7 +43,7 @@ type PruneImagesAsk struct { Limit int } -// declaredImages is the controller's `images` answer (novox/hq ADR 0251 §5). +// declaredImages is the mesh-artifact-sweep seat's `images` answer (novox/hq ADR 0251 §5). type declaredImages struct { Node string `json:"node"` Images []struct { @@ -129,7 +129,7 @@ func digestOf(ref string) string { return "" } -// askDeclared asks the controller what this machine's declarations name. An error means unknown. +// askDeclared asks the mesh-artifact-sweep seat what this machine's declarations name. An error means unknown. func (c *Client) askDeclared() (*declaredImages, error) { if c.Node == "" { return nil, errors.New("the runtime did not say which machine this is (MESH_NODE is empty)") @@ -137,20 +137,20 @@ func (c *Client) askDeclared() (*declaredImages, error) { if c.Ask == nil { return nil, errors.New("this bundle cannot ask the mesh") } - raw, err := c.Ask("seat:mesh-controller.images", map[string]any{"node": c.Node}) + raw, err := c.Ask("seat:mesh-artifact-sweep.images", map[string]any{"node": c.Node}) if err != nil { - return nil, fmt.Errorf("the controller's images verb did not answer: %w", err) + return nil, fmt.Errorf("the mesh-artifact-sweep seat's images verb did not answer: %w", err) } answer, output, ok := answerOf(raw) if !ok { - return nil, fmt.Errorf("the controller refused images: %s", firstLine(output)) + return nil, fmt.Errorf("the mesh-artifact-sweep seat refused images: %s", firstLine(output)) } var d declaredImages if err := json.Unmarshal(answer, &d); err != nil { - return nil, fmt.Errorf("the controller's images answer is not readable: %v", err) + return nil, fmt.Errorf("the mesh-artifact-sweep seat's images answer is not readable: %v", err) } if d.Node != "" && d.Node != c.Node { - return nil, fmt.Errorf("the controller answered for %q, not for this machine %q", d.Node, c.Node) + return nil, fmt.Errorf("the mesh-artifact-sweep seat answered for %q, not for this machine %q", d.Node, c.Node) } return &d, nil } diff --git a/modules/docker/cmd/docker-tools/prune_images_test.go b/modules/docker/cmd/docker-tools/prune_images_test.go index 62a5d6c7..df019fed 100644 --- a/modules/docker/cmd/docker-tools/prune_images_test.go +++ b/modules/docker/cmd/docker-tools/prune_images_test.go @@ -43,7 +43,7 @@ func imagesMachine() *fake { func declaring(t *testing.T, answer string) Asker { return func(key string, body any) (json.RawMessage, error) { - if key != "seat:mesh-controller.images" { + if key != "seat:mesh-artifact-sweep.images" { t.Errorf("asked %s", key) } if b, _ := body.(map[string]any); b["node"] != "laptop" { diff --git a/modules/docker/go.mod b/modules/docker/go.mod index 5faa536e..fe26ae06 100644 --- a/modules/docker/go.mod +++ b/modules/docker/go.mod @@ -1,5 +1,5 @@ module docker -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.6 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/docker/go.sum b/modules/docker/go.sum index 0dd60617..f8bfbd2b 100644 --- a/modules/docker/go.sum +++ b/modules/docker/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/docker/module.json b/modules/docker/module.json index fae954c3..500faea6 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -7,7 +7,7 @@ "privileged" ], "invokes": [ - "seat:mesh-controller.images" + "seat:mesh-artifact-sweep.images" ], "claims": [ { diff --git a/modules/gitea/cmd/forge-account-provider/builder_test.go b/modules/gitea/cmd/forge-account-provider/builder_test.go index f325acd7..8897013f 100644 --- a/modules/gitea/cmd/forge-account-provider/builder_test.go +++ b/modules/gitea/cmd/forge-account-provider/builder_test.go @@ -192,17 +192,14 @@ func (r *teamRig) pass() []provider.Outcome { return out } -// bind makes module on node bind gitea's forge-account in the assignments mirror. +// bind makes module on node bind gitea's forge-account in the seat mesh-provisioner's plans. func (r *teamRig) bind(node, module string) { - r.bus.set(provider.AssignmentsBucket, node+"."+module, []byte(fmt.Sprintf( - `{"node":%q,"module":%q,"provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s"}]}}`, - node, module, strings.ReplaceAll(node+"_"+module, "-", "_")))) + r.bus.bindPlan(node, module, "mesh_"+strings.ReplaceAll(node+"_"+module, "-", "_")) } +// unbind is the operator's yes to retiring module's forge-account on node (ADR 0570 clause 5). func (r *teamRig) unbind(node, module string) { - r.bus.mu.Lock() - delete(r.bus.buckets[provider.AssignmentsBucket], node+"."+module) - r.bus.mu.Unlock() + r.bus.retire(node, module) } // key reports a key in module's state keys, of the kind and for the account given. diff --git a/modules/gitea/cmd/forge-account-provider/bus_test.go b/modules/gitea/cmd/forge-account-provider/bus_test.go index 996a1816..7cacfe5b 100644 --- a/modules/gitea/cmd/forge-account-provider/bus_test.go +++ b/modules/gitea/cmd/forge-account-provider/bus_test.go @@ -30,6 +30,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" ) const bundleVar = "MESH_FORGE_ACCOUNT_TEST_BUNDLE" @@ -80,7 +81,7 @@ func startBus(t *testing.T) (*runtime, context.CancelFunc) { rt := &runtime{t: t, js: js, kvs: map[string]jetstream.KeyValue{}, ctx: ctx, wait: map[string]chan json.RawMessage{}} // The controller's job: a bucket per state the module declares (bound) or reads (the others). for state, bucket := range map[string]string{"bound": "gitea_bound", "ssh-client.keys": "ssh-client_keys", - "mesh-controller.assignments": "mesh-controller_assignments"} { + "mesh-provisioner.plans": "mesh-provisioner_plans", "mesh-provisioner.provisions": "mesh-provisioner_provisions"} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}) if err != nil { t.Fatal(err) @@ -296,9 +297,11 @@ func TestTheProviderWholeAnswersAMachineThatReportedItsKey(t *testing.T) { } rt, cancel := startBus(t) t.Cleanup(cancel) - rt.put("mesh-controller.assignments", "g14.ssh-client", map[string]any{"node": "g14", "module": "ssh-client", - "provisioning": map[string]any{"provisions": []map[string]any{{"provision": "forge-account", "provider": "novox", + rt.put("mesh-provisioner.plans", "g14.ssh-client", map[string]any{"v": 1, "node": "g14", "module": "ssh-client", + "plan": map[string]any{"provisions": []map[string]any{{"provision": "forge-account", "provider": "novox", "provider-module": "gitea", "as": "mesh_g14_ssh_client"}}}}) + rt.put("mesh-provisioner.provisions", "g14.ssh-client.forge-account", + json.RawMessage(provisionKey("g14", "ssh-client", provisions.Asked, nil))) rt.put("ssh-client.keys", "g14.nox", map[string]string{"node": "g14", "account": "nox", "kind": "machine", "public": pubA + " nox@g14 (mesh)", "fingerprint": "SHA256:x", "at": "2026-10-16T10:00:00Z"}) rt.start([]string{"MESH_NODE=novox", "MESH_MODULE=gitea", "MESH_SERVED_MODULE=gitea", diff --git a/modules/gitea/cmd/forge-account-provider/main_test.go b/modules/gitea/cmd/forge-account-provider/main_test.go index 024b5f6d..12583c16 100644 --- a/modules/gitea/cmd/forge-account-provider/main_test.go +++ b/modules/gitea/cmd/forge-account-provider/main_test.go @@ -25,7 +25,7 @@ func TestConfigFromEnvNamesWhatIsMissing(t *testing.T) { } // Issue 955: the runtime grants a module only the states its manifest `reads`, and the provider watches the machines' -// keys and, through the SDK's provider, the controller's assignments (where it learns its consumers). Run as the +// keys and, through the SDK's provider, the seat mesh-provisioner's plans and provisions (where it learns its consumers, ADR 0570). Run as the // bus test does, with every bucket made, a missing grant passes unseen; the manifest is read here instead. func TestTheManifestReadsEveryStateTheProviderWatches(t *testing.T) { raw, err := os.ReadFile("../../module.json") @@ -38,7 +38,7 @@ func TestTheManifestReadsEveryStateTheProviderWatches(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys"} { + for _, state := range []string{"mesh-provisioner.plans", "mesh-provisioner.provisions", "ssh-client.keys"} { if !slices.Contains(m.Reads, state) { t.Errorf("gitea's reads %v lack %q, so the runtime refuses its watch", m.Reads, state) } diff --git a/modules/gitea/cmd/forge-account-provider/service.go b/modules/gitea/cmd/forge-account-provider/service.go index 2e9716fb..0b2ad113 100644 --- a/modules/gitea/cmd/forge-account-provider/service.go +++ b/modules/gitea/cmd/forge-account-provider/service.go @@ -184,7 +184,9 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error { s.mu.Unlock() if !loaded { // After a restart the first pass can come before the keys arrive: nothing is removed or deleted on that. - return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted", source) + // Not ready, not failed: the entry waits and keeps serving what it served (provider.ErrNotReady). + return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted: %w", source, + provider.ErrNotReady) } have := s.keysFrom(source, node, kind) if len(have) == 0 { diff --git a/modules/gitea/cmd/forge-account-provider/service_test.go b/modules/gitea/cmd/forge-account-provider/service_test.go index 7f47709a..60242c91 100644 --- a/modules/gitea/cmd/forge-account-provider/service_test.go +++ b/modules/gitea/cmd/forge-account-provider/service_test.go @@ -17,6 +17,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" "gitea/internal/forgehttp" ) @@ -78,6 +79,22 @@ func (b *fakeBus) entry(t *testing.T, key string) provider.Entry { } return e } + +// serves is whether gitea's entry under key serves its consumer: there, in state (provided when none is given) and with +// values (ADR 0570 clause 3: an entry the provider could not make is kept, failed, serving nothing; one it is not +// ready to make waits and keeps serving). +func (b *fakeBus) serves(key string, state ...string) bool { + want := provisions.Provided + if len(state) > 0 { + want = state[0] + } + b.mu.Lock() + raw, ok := b.buckets[provider.BucketOf("gitea")][key] + b.mu.Unlock() + var e provider.Entry + return ok && json.Unmarshal(raw, &e) == nil && e.State == want && len(e.Values) > 0 +} + func (b *fakeBus) has(key string) bool { b.mu.Lock() defer b.mu.Unlock() @@ -85,11 +102,41 @@ func (b *fakeBus) has(key string) bool { return ok } -// bindClient makes ssh-client on node bind gitea's forge-account in the assignments mirror. +// bindClient makes ssh-client on node bind gitea's forge-account in the seat mesh-provisioner's plans. func (b *fakeBus) bindClient(node string) { - b.set(provider.AssignmentsBucket, node+".ssh-client", []byte(fmt.Sprintf( - `{"node":%q,"module":"ssh-client","provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s_ssh_client"}]}}`, - node, node))) + b.bindPlan(node, "ssh-client", "mesh_"+node+"_ssh_client") +} + +// bindPlan makes module on node bind gitea's forge-account as the seat mesh-provisioner keeps it: its plan, and the +// provision key asked of gitea on novox (novox/hq ADR 0570 clauses 1 to 3). +func (b *fakeBus) bindPlan(node, module, as string) { + b.set(provisions.PlansBucket, node+"."+module, []byte(fmt.Sprintf( + `{"v":1,"node":%q,"module":%q,"plan":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":%q}]}}`, + node, module, as))) + b.set(provisions.ProvisionsBucket, node+"."+module+".forge-account", provisionKey(node, module, provisions.Asked, nil)) +} + +// retire is the operator's yes to the retiring of module's forge-account on node: its plan gone, its key retiring +// with the warrant (ADR 0570 clause 5), the only way the provider lets an entry go. +func (b *fakeBus) retire(node, module string) { + b.mu.Lock() + delete(b.buckets[provisions.PlansBucket], node+"."+module) + b.mu.Unlock() + b.set(provisions.ProvisionsBucket, node+"."+module+".forge-account", provisionKey(node, module, provisions.Retiring, + &provisions.Warrant{Ask: "ask-1", Option: "yes", By: "operator", At: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)})) +} + +func provisionKey(node, module, state string, w *provisions.Warrant) []byte { + pr := provisions.Provision{Provision: "forge-account", State: state, Warrant: w, + Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "gitea"}} + if state == provisions.Retiring { + pr.Why = "no want needs it" + } + raw, err := provisions.Encode(pr) + if err != nil { + panic(err) + } + return raw } // fakeForge is gitea's API as far as the provider uses it. @@ -343,9 +390,7 @@ func TestAConsumerThatNoLongerBindsLosesItsMeshKeysAndKeepsItsAccount(t *testing r.forge.mu.Lock() r.forge.held["nox-agent"] = append(r.forge.held["nox-agent"], map[string]any{"id": int64(99), "title": "my own", "key": pubB}) r.forge.mu.Unlock() - r.bus.mu.Lock() - delete(r.bus.buckets[provider.AssignmentsBucket], "g14.ssh-client") - r.bus.mu.Unlock() + r.bus.retire("g14", "ssh-client") r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("the entry stayed") @@ -460,7 +505,7 @@ func TestADeletedKeyEntryRemovesTheMeshKeyAndTheBoundEntryButNotTheOperatorsKey( if got := r.forge.titles("nox-agent"); len(got) != 1 || got[0] != "my own" { t.Fatalf("keys = %v", got) } - if r.bus.has("g14.ssh-client.forge-account") { + if r.bus.serves("g14.ssh-client.forge-account") { t.Fatal("the entry still serves a machine whose key is gone") } } @@ -482,8 +527,8 @@ func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) { if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "mesh:g14.nox,my own" { t.Fatalf("keys = %s", got) } - if !r.bus.has("g14.ssh-client.forge-account") { - t.Fatal("the entry was deleted before the keys were read") + if !r.bus.serves("g14.ssh-client.forge-account", provisions.Waits) { + t.Fatal("the entry stopped serving before the keys were read; it waits and keeps serving") } if len(out) != 1 || !strings.Contains(out[0].Why, "not read yet") { t.Fatalf("the wait should be said: %+v", out) @@ -493,7 +538,7 @@ func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) { r.svc.loaded = true r.svc.mu.Unlock() r.pass() - if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "my own" || r.bus.has("g14.ssh-client.forge-account") { + if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "my own" || r.bus.serves("g14.ssh-client.forge-account") { t.Fatalf("keys = %s", got) } } @@ -515,7 +560,7 @@ func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) { r.svc.KeepRunning(ctx, func(ctx context.Context) error { calls++ if calls <= 2 { - return errors.New(`watching mesh-controller_assignments: gitea reads no state called "mesh-controller.assignments"`) + return errors.New(`watching mesh-provisioner_plans: gitea reads no state called "mesh-provisioner.plans"`) } close(serving) <-ctx.Done() @@ -535,7 +580,7 @@ func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) { if len(pauses) != 2 || pauses[0] != 5*time.Second || pauses[1] != 10*time.Second { t.Fatalf("the backoff is %v", pauses) } - if len(said) == 0 || !strings.Contains(said[0], "mesh-controller.assignments") { + if len(said) == 0 || !strings.Contains(said[0], "mesh-provisioner.plans") { t.Fatalf("a refusal is said loudly: %v", said) } loud := 0 diff --git a/modules/gitea/cmd/forge-account-provider/watch_test.go b/modules/gitea/cmd/forge-account-provider/watch_test.go index 2fb60b97..60618da4 100644 --- a/modules/gitea/cmd/forge-account-provider/watch_test.go +++ b/modules/gitea/cmd/forge-account-provider/watch_test.go @@ -142,7 +142,7 @@ func TestABuildModulesStateThatNeverAnswersHoldsBackOnlyItsOwnEntries(t *testing } // 1482/AC4 (the provider's half): the manifest reads the states of both build modules beside ssh-client's and the -// controller's assignments; the provider watches exactly the modules the code names as builders. +// seat mesh-provisioner's plans; the provider watches exactly the modules the code names as builders. func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -154,7 +154,7 @@ func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} { + for _, state := range []string{"mesh-provisioner.plans", "mesh-provisioner.provisions", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} { if !slices.Contains(m.Reads, state) { t.Errorf("gitea's reads %v lack %q", m.Reads, state) } diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index 6639c1c3..87ac68dc 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -3,7 +3,7 @@ module gitea go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/modules/gitea/go.sum b/modules/gitea/go.sum index 2a3e9f34..a2b8e018 100644 --- a/modules/gitea/go.sum +++ b/modules/gitea/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 3d9d00f6..2b197288 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -400,6 +400,7 @@ "provides": [ { "name": "npm-package-registry", + "credential": {"form": "pair"}, "scope": "mesh", "identity": { "max": 40, @@ -408,6 +409,7 @@ }, { "name": "git", + "credential": {"form": "none"}, "scope": "mesh" }, { @@ -420,6 +422,7 @@ }, { "name": "forge-account", + "credential": {"form": "none"}, "scope": "mesh" } ], @@ -564,7 +567,8 @@ "announced" ], "reads": [ - "mesh-controller.assignments", + "mesh-provisioner.plans", + "mesh-provisioner.provisions", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", diff --git a/modules/go/module.json b/modules/go/module.json index 48a6bf57..9783eda4 100644 --- a/modules/go/module.json +++ b/modules/go/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "go-toolchain", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "resources": [ diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index 53389a5f..13a7905f 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -7,6 +7,9 @@ "scope": "mesh", "identity": { "in": "an InfluxDB v1 authorization" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go b/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go index 20843dcd..80f02f9a 100644 --- a/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go +++ b/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/provider" "git.novox.be/novox/mesh-sdk/go/provider/harness" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // grantGives writes a grant file (MESH_RECEIVES) naming consumers the plans do not move (issue 937): each one's @@ -38,11 +39,33 @@ func grantGives(t *testing.T, given ...map[string]any) string { return at } -// noOpPlan puts one assignment the bus can read that binds nothing of keycloak's, so the moved path has plans it +// noOpPlan puts one plan the bus can read that binds nothing of keycloak's, so the moved path has plans it // could read (m.asked answers "read", issue 241) even when nothing is moved. func noOpPlan(bus *bucketBus, node, module string) { - raw, _ := json.Marshal(map[string]any{"node": node, "module": module}) - _ = bus.Put(ctx, provider.AssignmentsBucket, node+"."+module, raw) + seatPlan(bus, node, module, map[string]any{}) +} + +// seatPlan puts a module's plan as the seat mesh-provisioner keeps it, and for each of its provisions served by +// keycloak on novox the key the seat composes, asked (novox/hq ADR 0570 clauses 1 to 3). +func seatPlan(bus *bucketBus, node, module string, plan map[string]any) { + raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "plan": plan}) + _ = bus.Put(ctx, provisions.PlansBucket, node+"."+module, raw) + list, _ := plan["provisions"].([]any) + for _, e := range list { + entry := e.(map[string]any) + name, _ := entry["provision"].(string) + local, _ := entry["local"].(string) + key, err := provisions.Key(node, module, name, local) + if err != nil { + panic(err) + } + asked, err := provisions.Encode(provisions.Provision{Provision: name, Local: local, State: provisions.Asked, + Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "keycloak"}}) + if err != nil { + panic(err) + } + _ = bus.Put(ctx, provisions.ProvisionsBucket, key, asked) + } } // waitUntil polls cond until it is true or the timeout passes. diff --git a/modules/keycloak/cmd/keycloak-provider/live_retire_test.go b/modules/keycloak/cmd/keycloak-provider/live_retire_test.go index 4cbc24a0..0ab5cc34 100644 --- a/modules/keycloak/cmd/keycloak-provider/live_retire_test.go +++ b/modules/keycloak/cmd/keycloak-provider/live_retire_test.go @@ -62,6 +62,14 @@ func TestLiveRetirement(t *testing.T) { if code := authorize(); code == 200 { t.Fatal("a retired client is still served") } + // Its secret is removed (ADR 0570 clause 5): the mesh's no longer authenticates. + if found, err := o.KC.FindClient(ctx, o.Realm, p.As); err != nil || found == nil { + t.Fatal(found, err) + } else if id, _ := found["id"].(string); true { + if secret, err := o.KC.ClientSecretByID(ctx, o.Realm, id); err != nil || secret == p.Password { + t.Fatal("a retired client keeps the mesh's secret", err) + } + } // Retired, so a person may delete it. if err := o.Delete(ctx, p.As); err != nil { t.Fatal(err) diff --git a/modules/keycloak/cmd/keycloak-provider/moved_test.go b/modules/keycloak/cmd/keycloak-provider/moved_test.go index f69aa363..dd5371dd 100644 --- a/modules/keycloak/cmd/keycloak-provider/moved_test.go +++ b/modules/keycloak/cmd/keycloak-provider/moved_test.go @@ -155,10 +155,9 @@ func TestBothPathsLeaveTheSameClientInKeycloak(t *testing.T) { t.Fatal(err) } bus := &bucketBus{buckets: map[string]map[string][]byte{}} - plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana", "provisioning": map[string]any{ + seatPlan(bus, "home", "grafana", map[string]any{ "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", - "provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}}}) - _ = bus.Put(ctx, provider.AssignmentsBucket, "home.grafana", plan) + "provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}}) pr := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: bus, Vault: provider.ConsumersFile{Paths: []string{file}}, Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, diff --git a/modules/keycloak/cmd/keycloak-provider/oidc.go b/modules/keycloak/cmd/keycloak-provider/oidc.go index c7815462..ca702b0a 100644 --- a/modules/keycloak/cmd/keycloak-provider/oidc.go +++ b/modules/keycloak/cmd/keycloak-provider/oidc.go @@ -24,15 +24,18 @@ package main import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/provider/harness" "net/url" "regexp" "sort" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/provider/harness" ) // Mark is the attribute marking a client as the mesh's own work. @@ -307,8 +310,10 @@ func (o OidcClients) Holds(ctx context.Context, p harness.Provision) (bool, erro return secret == p.Password, nil } -// Retire disables a consumer's client — only one the mesh made — and marks it with when and why. -// Answers what happened: "retired", "absent" or "not ours". +// Retire, on the operator's yes (novox/hq ADR 0570 clause 5), removes the client's secret — replaced by one nobody +// is given, so the mesh's credential no longer authenticates — and disables and marks the client with when and why. +// The client and its settings are kept; asked for again, Ensure sets the mesh's secret and enables it. Only a client +// the mesh made. Answers what happened: "retired", "absent" or "not ours". func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) (string, error) { found, err := o.KC.FindClient(ctx, o.Realm, as) if err != nil { @@ -332,6 +337,11 @@ func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) ( attrs[MarkRetiredWhy] = why merged["attributes"] = attrs merged["enabled"] = false + unknown := make([]byte, 32) + if _, err := rand.Read(unknown); err != nil { + return "", err + } + merged["secret"] = hex.EncodeToString(unknown) // never kept, never said id, _ := found["id"].(string) return "retired", o.KC.UpdateClient(ctx, o.Realm, id, merged) } diff --git a/modules/keycloak/cmd/keycloak-provider/oidc_test.go b/modules/keycloak/cmd/keycloak-provider/oidc_test.go index 45c95ae3..4d8f52d0 100644 --- a/modules/keycloak/cmd/keycloak-provider/oidc_test.go +++ b/modules/keycloak/cmd/keycloak-provider/oidc_test.go @@ -186,7 +186,7 @@ func TestAClientTheMeshDidNotMakeIsRefusedAndLeftAlone(t *testing.T) { } } -// Retired is disabled and marked, everything else kept; asked for again it is enabled as it was; only +// Retired is its secret removed, disabled and marked, everything else kept (ADR 0570 clause 5); asked for again it is enabled as it was; only // a deletion removes it, and never while enabled (novox/hq ADR 0230). func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) { f, o := oidcWorld(t) @@ -204,7 +204,7 @@ func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) { t.Fatal(done, err) } c := f.clients[id] - if c["enabled"] != false || c["secret"] != "s" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" || + if c["enabled"] != false || c["secret"] == "s" || c["secret"] == "" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" || attributes(c)[MarkRetiredWhy] != "the mesh stopped asking for it" || c["description2"] == nil { t.Fatalf("%v", c) } @@ -219,7 +219,7 @@ func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) { t.Fatal(err) } c = f.clients[id] - if c["enabled"] != true || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil { + if c["enabled"] != true || c["secret"] != "s" || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil { t.Fatalf("%v", c) } mustHold(t, o, p, true) diff --git a/modules/keycloak/cmd/keycloak-provider/provisioner.go b/modules/keycloak/cmd/keycloak-provider/provisioner.go index e0428a4e..6bf4aef6 100644 --- a/modules/keycloak/cmd/keycloak-provider/provisioner.go +++ b/modules/keycloak/cmd/keycloak-provider/provisioner.go @@ -62,7 +62,8 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error { return nil } -// Retire disables the consumer's client and marks it, never deletes it (novox/hq ADR 0230). +// Retire removes the consumer's client secret and disables and marks the client, never deletes it (novox/hq ADR 0570 +// clause 5, ADR 0230). func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error { if err := a.refused(); err != nil { return err @@ -75,7 +76,7 @@ func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, wh case "not ours": a.log("[provisioner:oidc-client] %s: a client of that id exists that the mesh did not make — left alone", as) case "retired": - a.log("[provisioner:oidc-client] retired client %s in realm %s: disabled, kept, marked to delete", as, a.clients.Realm) + a.log("[provisioner:oidc-client] retired client %s in realm %s: its secret removed, disabled, kept, marked to delete", as, a.clients.Realm) a.announce("client.retired", map[string]any{"realm": a.clients.Realm, "clientId": as}) } return nil diff --git a/modules/keycloak/go.mod b/modules/keycloak/go.mod index 28481201..64fffc79 100644 --- a/modules/keycloak/go.mod +++ b/modules/keycloak/go.mod @@ -2,4 +2,4 @@ module keycloak go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/keycloak/go.sum b/modules/keycloak/go.sum index 6162079d..f8bfbd2b 100644 --- a/modules/keycloak/go.sum +++ b/modules/keycloak/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/keycloak/module.json b/modules/keycloak/module.json index 847d5f4f..84c08f10 100644 --- a/modules/keycloak/module.json +++ b/modules/keycloak/module.json @@ -9,6 +9,7 @@ "provides": [ { "name": "oidc-client", + "credential": {"form": "pair"}, "scope": "mesh", "identity": { "max": 255, @@ -240,6 +241,7 @@ "bound" ], "reads": [ - "mesh-controller.assignments" + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ] } diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 93d72adb..0faccc58 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -665,7 +665,14 @@ "path": "${dir:mesh-state}/config.json", "mode": "0600", "template": "files/config.json", - "takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"], + "takes-config": [ + "domain", + "proxy-address", + "sitename", + "website", + "url", + "container" + ], "merge": "json" }, { @@ -758,6 +765,9 @@ "identity": { "max": 64, "in": "a mailbox's local part" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/mesh-build-image/.gitignore b/modules/mesh-build-image/.gitignore new file mode 100644 index 00000000..00eb03a8 --- /dev/null +++ b/modules/mesh-build-image/.gitignore @@ -0,0 +1 @@ +mesh-build-image diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 701791be..cb908ec3 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index 9dde7f0c..a2b8e018 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-node/.gitignore b/modules/mesh-build-node/.gitignore new file mode 100644 index 00000000..f7604aaa --- /dev/null +++ b/modules/mesh-build-node/.gitignore @@ -0,0 +1 @@ +mesh-build-node diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index f8b463c0..cd83b38b 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index eee5d03f..5f0e39b2 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-catalog/heard.ts b/modules/mesh-catalog/heard.ts index eaa8f882..619639ed 100644 --- a/modules/mesh-catalog/heard.ts +++ b/modules/mesh-catalog/heard.ts @@ -45,13 +45,72 @@ export interface Built { branches?: string[]; } -/** The events a build outcome reaches this module on: the build seat's, as it happens, and the control plane's replay. */ -export const BUILD_EVENTS = [ - // The build seat node-build-agent and the retired mesh-build-machine went with build-agent (novox/hq ADR 0524 §5, - // ADR 0525 §10): no build seat says `built` since R4g, so what the mesh holds is asked when this module starts. - // What the mesh already held, when this module asks what it missed (novox/hq ADR 0134). - "mesh-controller.built-before", -] as const; +/** + * Where a build outcome comes from now: the builder's log (novox/hq ADR 0571 clauses 1 and 9). The module asks the + * builder's tool for the builds that ended since a time, and again whenever the builder's state of done keys changes. + */ +export const BUILDER = "mesh-delivery-builder"; +export const BUILDS_SINCE_TOOL = "builder_builds_since"; +export const BUILDS_STATE = "mesh-delivery-builder.builds"; +export const MANIFESTS_STATE = "mesh-delivery-indexer.manifests"; + +/** One entry of the builder's log, an ended build (mesh-sdk delivery.BuildLogEntry). */ +export interface LogEntry { + event: string; + module: string; + fp: string; + repository: string; + directory: string; + commit: string; + ref?: string; + node?: string; + at: string; + /** done, failed or queued. */ + outcome?: string; + /** The digest of the manifest built, read from the indexer's state of manifests. */ + manifest?: string; + against?: string[]; + made?: Made[]; +} + +/** The answer of the builder's tool: every ended build since a time, oldest first, and whether the log held more. */ +export interface BuildsSince { + since: string; + builds: LogEntry[]; + more?: boolean; +} + +/** + * What a tool answers, whether bare or wrapped as the runtime wraps a tool's reply ({output} or {content:[{text}]}). + */ +export function answerOf(raw: unknown): T { + const r = raw as { output?: unknown; content?: { text?: string }[] } | null; + if (r && typeof r.output === "string") return JSON.parse(r.output) as T; + if (r && Array.isArray(r.content) && typeof r.content[0]?.text === "string") return JSON.parse(r.content[0].text) as T; + if (typeof raw === "string") return JSON.parse(raw) as T; + return raw as T; +} + +/** + * A log entry as the build outcome the catalogue places: a build that is done is placed with its manifest, what it + * made and what it was built against; one that failed is passed over (placeable says so); one still queued has not + * ended and is not taken in. `replay` is set by the caller for history. + */ +export function builtFrom(e: LogEntry, manifest: unknown, replay: boolean): Built | undefined { + if (e.outcome === "queued") return undefined; + return { + module: e.module, + commit: e.commit, + repository: e.repository, + path: e.directory, + ref: e.ref ?? "", + manifest, + made: e.made ?? [], + against: e.against ?? [], + replay, + ...(e.outcome === "failed" ? { failed: "the builder's log says the build failed" } : {}), + }; +} /** * Whether a build outcome is a module-version to place in the graph, and why not when it is not. diff --git a/modules/mesh-catalog/index.ts b/modules/mesh-catalog/index.ts index 13a98d2d..cc034002 100644 --- a/modules/mesh-catalog/index.ts +++ b/modules/mesh-catalog/index.ts @@ -9,9 +9,15 @@ // and appear the next time round, once whatever they were waiting for is registered. A chain and a // diamond need no special handling, and nothing holds a plan. -import { on, emit } from "@novox/mesh-sdk/events"; +import { emit } from "@novox/mesh-sdk/events"; +import { broker } from "@novox/mesh-sdk/messaging"; +import { invokeTool } from "@novox/mesh-sdk/tools"; +import { state } from "@novox/mesh-sdk/state"; import { Graph } from "./store.js"; -import { BUILD_EVENTS, placeable, type Built } from "./heard.js"; +import { + answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE, + type Built, type BuildsSince, type LogEntry, +} from "./heard.js"; const graph = Graph.fromEnv(); @@ -77,18 +83,74 @@ const placeTheBuild = async (event: { body: unknown }): Promise => { } }; -// As it happens — from the build seat that builds now, and the retired one while it still answers — and -// what the mesh already held when this module asked what it missed (novox/hq issue 712). -for (const heard of BUILD_EVENTS) await on(heard, placeTheBuild); +// **From the builder's log** (novox/hq ADR 0571 clauses 1 and 9). The builder keeps every build's end in a log of its +// own and answers the builds that ended since a time by its tool. This module asks at its start for what it missed (all +// the log holds, taken in as history: registered, announced to nobody), and again whenever the builder's state of done +// keys changes, for what ended since the last build it took in. Registering a build already held changes nothing, and a +// build is taken in once per run of this module. +const taken = new Set(); +let since = new Date(0).toISOString(); +let caughtUp = false; +let asking: Promise | undefined; +let askAgain = false; -// **And ask for what was built before this catalogue existed** (novox/hq 04-ISSUES/050). -// -// The queue above is durable, so nothing is missed once this is running. What it cannot have is -// what was announced before it first ran — and on a fresh mesh that is never arbitrary: the shared -// base, the store this runs on, and this module itself are each necessarily built BEFORE a -// catalogue exists to hear about them. The graph's foundation is the part it never sees. -// -// Asked on every start, not only the first. A catalogue cannot tell whether it has a gap, and the -// answer is idempotent: registering a build already held changes nothing and announces nothing. -// Asked AFTER subscribing, so a build arriving during the replay is not lost between the two. +const manifestOf = async (digest: string | undefined): Promise => { + if (!digest) return {}; + const held = await state<{ manifest?: string }>(MANIFESTS_STATE).get(digest); + if (!held?.value.manifest) return {}; + return JSON.parse(held.value.manifest); +}; + +const takeIn = async (e: LogEntry, replay: boolean): Promise => { + const key = `${e.module}/${e.fp}`; + if (taken.has(key)) return; + const built = builtFrom(e, await manifestOf(e.manifest), replay); + if (!built) return; + await placeTheBuild({ body: built }); + taken.add(key); +}; + +const catchUp = async (): Promise => { + for (;;) { + const answer = answerOf( + await invokeTool(broker(), BUILDER, BUILDS_SINCE_TOOL, { since }), + ); + for (const e of answer.builds ?? []) { + await takeIn(e, !caughtUp); + if (e.at > since) since = e.at; + } + if (!answer.more) break; + } + caughtUp = true; +}; + +/** One ask at a time; a change heard while asking is asked for again once, so none is missed. */ +const ask = (): Promise => { + if (asking) { + askAgain = true; + return asking; + } + asking = (async () => { + try { + do { + askAgain = false; + await catchUp(); + } while (askAgain); + } catch (err) { + console.error("mesh-catalog: the builder's builds since", since, "could not be taken in:", err); + } finally { + asking = undefined; + } + })(); + return asking; +}; + +// Asked AFTER watching begins, so a build ending while the first ask runs asks again rather than being missed: the watch +// hands every current key first (the start's ask), then every change. +await state(BUILDS_STATE).watch(async (change) => { + if (!change.current) await ask(); +}); +await ask(); + +// **And say that this catalogue has caught up** (novox/hq 04-ISSUES/050). await emit("catching-up", {}); diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json index b8f45096..c5ad61a4 100644 --- a/modules/mesh-catalog/module.json +++ b/modules/mesh-catalog/module.json @@ -25,9 +25,7 @@ "secrets": { "postgres-database": "${dir:state}/database.secret" }, - "consumes": [ - "mesh-controller.built-before" - ], + "consumes": [], "emits": [ "registered", "upgraded", @@ -92,5 +90,12 @@ "from": "files" } ] - } + }, + "invokes": [ + "mesh-delivery-builder.builder_builds_since" + ], + "reads": [ + "mesh-delivery-builder.builds", + "mesh-delivery-indexer.manifests" + ] } diff --git a/modules/mesh-catalog/test/heard.test.ts b/modules/mesh-catalog/test/heard.test.ts index 84b91ac4..391cb5e9 100644 --- a/modules/mesh-catalog/test/heard.test.ts +++ b/modules/mesh-catalog/test/heard.test.ts @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; -import { BUILD_EVENTS, placeable } from "../heard.js"; +import { answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE } from "../heard.js"; // A walk's build as the build seat says it (mesh-controller's link.BuildResult), here mailu's of novox/hq issue 712. const walkBuild = { @@ -20,14 +20,25 @@ const walkBuild = { made: [{ name: "code", kind: "bundle", reference: "artifact-store://mailu/code/blobs/sha256:d041" }], }; -test("the catalogue hears no retired build seat, and what the mesh held (issue 712, issue 1887 E2)", async () => { - for (const retired of ["node-build-agent.built", "mesh-build-machine.built"]) { - assert.ok(!(BUILD_EVENTS as readonly string[]).includes(retired), `${retired} is still heard`); - } - assert.ok(BUILD_EVENTS.includes("mesh-controller.built-before")); - // Subscribing is not enough: the mesh grants and routes only what the manifest consumes. +test("the catalogue takes builds from the builder's log, and hears no controller event (ADR 0571 clause 9)", async () => { const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); - for (const heard of BUILD_EVENTS) assert.ok(manifest.consumes.includes(heard), `${heard} is not consumed`); + assert.deepEqual(manifest.consumes, []); + assert.ok(manifest.invokes.includes(`${BUILDER}.${BUILDS_SINCE_TOOL}`), "the builder's tool is not invoked"); + for (const read of [BUILDS_STATE, MANIFESTS_STATE]) assert.ok(manifest.reads.includes(read), `${read} is not read`); + // A log entry is placed as a build: done with its manifest, made and against; failed passed over; queued not taken in. + const entry = { event: "build.ended", module: "mailu", fp: "fp1", repository: walkBuild.repository, directory: "modules/mailu", + commit: walkBuild.commit, ref: "main", at: "2026-10-29T20:00:00Z", outcome: "done", manifest: "ab", against: ["img@sha256:1"], + made: walkBuild.made }; + const done = builtFrom(entry, walkBuild.manifest, false); + assert.deepEqual(placeable(done!), { placed: "yes", module: "mailu", commit: walkBuild.commit }); + assert.deepEqual([done!.path, done!.against, done!.replay], ["modules/mailu", ["img@sha256:1"], false]); + assert.equal(placeable(builtFrom({ ...entry, outcome: "failed" }, {}, true)!).placed, "no"); + assert.equal(builtFrom({ ...entry, outcome: "queued" }, {}, true), undefined); + // The tool's answer is read bare or as the runtime wraps it. + const bare = { since: "x", builds: [entry] }; + for (const raw of [bare, { output: JSON.stringify(bare) }, { content: [{ text: JSON.stringify(bare) }] }]) { + assert.equal(answerOf(raw).builds[0].module, "mailu"); + } }); test("a walk's build is placed, under its module and commit", () => { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/compare.go b/modules/mesh-conditions/cmd/mesh-conditions/compare.go new file mode 100644 index 00000000..e04e805a --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/compare.go @@ -0,0 +1,221 @@ +package main + +// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare` +// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the +// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and +// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a +// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the +// controller until the lease write, and is refused after it. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Difference is one key the two sides do not agree on, or one listed apart. +type Difference struct { + Key string `json:"key"` + Kind string `json:"kind"` + Reason string `json:"reason"` +} + +// Comparison is compare's answer. +type Comparison struct { + Differences []Difference `json:"differences"` + // Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held. + Apart []Difference `json:"apart"` + Same int `json:"same"` + At time.Time `json:"at"` +} + +// Empty says nothing differs. +func (c Comparison) Empty() bool { return len(c.Differences) == 0 } + +// Compare is the keeper's open conditions against the controller's. +func (k *Keeper) Compare() (Comparison, error) { + mine, err := k.Open() + if err != nil { + return Comparison{}, err + } + k.mu.Lock() + k.init() + theirs := map[string]health.Condition{} + for key, c := range k.ctrl { + theirs[key] = c + } + heldNow := map[string]held{} + for key, h := range k.held { + heldNow[key] = h + } + k.mu.Unlock() + out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()} + open := map[string]health.Condition{} + for _, c := range mine { + open[c.Key] = c + } + keys := map[string]bool{} + for key := range open { + keys[key] = true + } + for key := range theirs { + keys[key] = true + } + for key := range heldNow { + keys[key] = true + } + sorted := make([]string, 0, len(keys)) + for key := range keys { + sorted = append(sorted, key) + } + sort.Strings(sorted) + for _, key := range sorted { + c, inMine := open[key] + t, inTheirs := theirs[key] + kind := c.Kind + if !inMine { + kind = t.Kind + } + home, homed := k.homeOf(kind) + if h, ok := heldNow[key]; ok { + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))}) + continue + } + switch { + case homed && (home.Carried || home.Replaced): + why := "ends with the controller: carried from its bucket until the lease write" + if home.Replaced { + why = "ends with the controller: its fault is observed otherwise" + } + if home.Why != "" { + why += " (" + home.Why + ")" + } + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why}) + case inMine && !inTheirs && (!homed || !home.Controller): + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"}) + case inMine && !inTheirs: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: "open at the keeper, not at the controller"}) + case !inMine && !homed: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)}) + case !inMine: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)}) + case c.Severity != t.Severity: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)}) + default: + out.Same++ + } + } + return out, nil +} + +// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands. +type Asks interface { + Ask(a asks.Ask) error +} + +// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write. +func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a take-over says why") + } + if k.LeaseAway() { + return nil, errors.New("the controller's lease is handed away: the keeper is the one source already") + } + cmp, err := k.Compare() + if err != nil { + return nil, err + } + if !cmp.Empty() { + return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none", + len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason) + } + now := k.Now().UTC() + id := fmt.Sprintf("take-over-%d", now.UnixNano()) + act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id} + bind, err := asks.ActDigest(act) + if err != nil { + return nil, err + } + ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator, + Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+ + "the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+ + "gives up its lease.", cmp.Same), + Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}}, + Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.", + About: Module + "-take-over", Details: "Asked by " + by + ": " + why} + if err := a.Ask(ask); err != nil { + return nil, err + } + return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil +} + +// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper. +func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error { + if o.ID != "yes" { + k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID) + return nil + } + cmp, err := k.Compare() + if err != nil { + return err + } + if !cmp.Empty() { + return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask, + len(cmp.Differences)) + } + return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask}) +} + +// HandBack switches the readers back to the controller, until the lease write. +func (k *Keeper) HandBack(by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a hand-back says why") + } + if k.LeaseAway() { + return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to") + } + t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why} + if err := k.writeTakeover(t); err != nil { + return nil, err + } + return map[string]any{"follow": t.Follow, "at": t.At}, nil +} + +// writeTakeover writes the take-over key, and keeps it in history as the record of the switch. +func (k *Keeper) writeTakeover(t health.Takeover) error { + raw, err := health.EncodeTakeover(t) + if err != nil { + return err + } + _, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return err + } + if found { + _, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev) + } else { + _, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw)) + } + if errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("the take-over key moved under this write: read it again and ask again") + } + if err != nil { + return err + } + k.Takeover(t) + k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By) + return nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go new file mode 100644 index 00000000..8491812e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -0,0 +1,340 @@ +package main + +// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12; +// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a +// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or +// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state +// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once, +// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's +// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery +// did not end. A finding about another module or about the machine itself is never held. A condition already open is +// not held: it is no news. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The modules whose delivery holds what they serve beyond their own findings (Limits 12). +const ( + controllerModule = "mesh-controller" + runtimeModule = "node-tools" +) + +// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its +// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names. +var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true} + +// about is a node and a module a finding or a condition is about, for the hold. +type about struct{ Node, Module string } + +// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its +// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is +// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself. +func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about { + var out []about + parts := strings.Split(s.ID, ".") + switch s.Scope { + case health.ScopeModule: + if len(parts) >= 2 { + out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) + } + case health.ScopeProvider: + if len(parts) >= 2 { + out = append(out, about{Node: parts[1], Module: parts[0]}) + } + case health.ScopeSeat: + // The module the placement says holds the seat on that node, or the seat's own name where none is read. + if len(parts) >= 2 { + node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".") + module := k.holders[node+"."+seat] + if module == "" { + module = seat + } + out = append(out, about{Node: node, Module: module}) + } + case health.ScopeMachine: + // The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is + // delivered. + if source == "S11" || kind == "tools-silent" { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + if s.Machine == "" && len(parts) > 0 { + out[len(out)-1].Node = parts[0] + } + } + return out + } + if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) { + for hk := range k.wants { + if node, module := splitNodeModule(hk); module == controllerModule { + out = append(out, about{Node: node, Module: module}) + } + } + } + if s.Machine != "" && s.Scope != health.ScopeMachine { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + } + return out +} + +func splitNodeModule(hk string) (string, string) { + node, module, _ := strings.Cut(hk, ".") + return node, module +} + +// motion is the delivery of module on node, from its want and have. +func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) { + w, wanted := k.wants[node+"."+module] + h, has := k.haves[node+"."+module] + return delivery.MotionOf(w, wanted, h, has, "", false), w +} + +// holdFor says whether key is held now, and as what. Called with mu held. +func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) { + if open || (f == nil && carried == nil) { + return held{}, false + } + var kind, source string + var subject health.Subject + if f != nil { + kind, source, subject = f.Kind, f.Source, f.Subject + } else { + kind, source, subject = carried.Kind, carried.Source, carried.Subject + } + if _, raisedOut := k.outOfHold[key]; raisedOut { + return held{}, false + } + for _, a := range k.aboutOf(kind, source, subject) { + m, w := k.motion(a.Node, a.Module) + if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP { + continue + } + since := k.Now().UTC() + if was, ok := k.held[key]; ok { + since = was.Since + } + return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true + } + return held{}, false +} + +// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once. +func (k *Keeper) keepHeld(h held) error { + k.mu.Lock() + _, was := k.held[h.Key] + k.held[h.Key] = h + bound := k.bound(h.Module) + k.mu.Unlock() + if was { + return nil + } + if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("keeping %s held: %w", h.Key, err) + } + b := boundKey(h.Node, h.Module) + err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version, + "bound-seconds": int(bound / time.Second)}, bound) + if err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err) + } + k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound) + return nil +} + +// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state +// `bounds` is written with it as its own time to live, and its expiry is the bound passing. +func (k *Keeper) bound(string) time.Duration { + if k.Bound > 0 { + return k.Bound + } + return delivery.DefaultDeliveryBound +} + +func boundKey(node, module string) string { return node + "." + module } + +// dropHeld lets a held key go: raised, or gone with why said. +func (k *Keeper) dropHeld(key, why string) { + k.mu.Lock() + h, ok := k.held[key] + delete(k.held, key) + k.mu.Unlock() + if !ok { + return + } + if err := k.Store.Delete(HeldState, key); err != nil { + k.Logf("the held %s could not be removed from held: %v", key, err) + } + if why != "" { + k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why) + } +} + +// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or +// drops what it held. +func (k *Keeper) Want(key string, value []byte, deleted bool) error { + node, module, err := delivery.ParseWantKey(key) + if err != nil { + return nil // not a want of a module on a node (a held fingerprint's key) + } + k.mu.Lock() + k.init() + if deleted { + delete(k.wants, node+"."+module) + } else if w, err := delivery.DecodeWant(value); err == nil { + k.wants[node+"."+module] = w + } else { + k.Logf("the want %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Have takes one change of a node's have. +func (k *Keeper) Have(key string, value []byte, deleted bool) error { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + return nil + } + k.mu.Lock() + k.init() + if deleted { + delete(k.haves, node+"."+module) + } else if h, err := delivery.DecodeHave(value); err == nil { + k.haves[node+"."+module] = h + } else { + k.Logf("the have %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Placement takes one change of the placement: which module holds which seat on a node. +func (k *Keeper) Placement(key string, value []byte, deleted bool) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + return + } + k.mu.Lock() + defer k.mu.Unlock() + k.init() + for hk := range k.holders { + if strings.HasPrefix(hk, node+".") { + delete(k.holders, hk) + } + } + if deleted { + return + } + p, err := delivery.DecodePlacement(value) + if err != nil { + k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err) + return + } + for _, h := range p.Holds { + k.holders[node+"."+h.Claim] = h.Module + } +} + +// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. +func (k *Keeper) BoundPassed(key string) error { + node, module := splitNodeModule(key) + k.mu.Lock() + k.init() + _, w := k.motion(node, module) + holding := false + for _, h := range k.held { + holding = holding || (h.Node == node && h.Module == module) + } + if !holding { + k.mu.Unlock() + return nil + } + k.passed[key] = w.FP + k.mu.Unlock() + return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node, + k.bound(module))) +} + +// judge raises or drops what node and module hold when their delivery ended, or with line past its bound. +func (k *Keeper) judge(node, module, line string) error { + k.mu.Lock() + m, w := k.motion(node, module) + var keys []string + for key, h := range k.held { + if h.Node == node && h.Module == module { + keys = append(keys, key) + } + } + if line == "" && !m.Ended() && m.Delivering { + k.mu.Unlock() + return nil + } + if line == "" { + line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node) + if m.Failed { + line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node) + } + delete(k.passed, node+"."+module) + } + for _, key := range keys { + k.outOfHold[key] = line + } + k.mu.Unlock() + sort.Strings(keys) + var problems []string + for _, key := range keys { + k.dropHeld(key, "") + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + k.mu.Lock() + delete(k.outOfHold, key) + k.mu.Unlock() + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +func short(fp string) string { + if _, rest, ok := strings.Cut(fp, ":"); ok { + fp = rest + } + if len(fp) > 12 { + return fp[:12] + } + return fp +} + +// Held is every key held now, for the read tool and compare. +func (k *Keeper) Held() []held { + k.mu.Lock() + defer k.mu.Unlock() + out := make([]held, 0, len(k.held)) + for _, h := range k.held { + out = append(out, h) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} + +// restoreHeld takes the state `held` as read at the start. +func (k *Keeper) restoreHeld(key string, value []byte) { + var h held + if json.Unmarshal(value, &h) != nil { + return + } + k.mu.Lock() + k.init() + k.held[key] = h + k.mu.Unlock() +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go new file mode 100644 index 00000000..fa668544 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves, +// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases +// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented +// order, not a capture; the values are captured. +const ( + ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c" + ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0" + ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6" + ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142" +) + +// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have. +// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still +// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a +// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a +// held key as held. +func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) { + want := func(fp, composed string) []byte { + raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed}) + return raw + } + have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte { + raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h}) + return raw + } + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"}) + for _, c := range []struct { + name string + // end is the have the delivery ends with, or nil for a delivery past its bound. + end []byte + standing bool + said string + line string + }{ + {name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)}, + {name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK), + standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"}, + {name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"}, + {name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true, + said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"}, + } { + r := newRig() + _ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + // The delivery starts: the assigner wants the next fingerprint; the node applies it. + _ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false) + own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now) + other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now) + machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now) + for _, f := range []struct { + key string + raw []byte + }{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} { + if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil { + t.Fatal(err) + } + } + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + // The delivery's bound is its own key's time to live, the module's bound (the default where it names none). + if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound { + t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"]) + } + if got := r.changes(); got != "raised "+other+", raised "+machine { + t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got) + } + // C8: compare lists the held keys as held, never as a difference. + cmp, _ := r.k.Compare() + held := 0 + for _, d := range cmp.Apart { + if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") { + held++ + } + } + if held != 2 { + t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp) + } + // The carried condition about the controller goes within the delivery; its own finding goes unless it stands. + _ = r.k.Controller("core.controller.deaf", nil, true) + if !c.standing { + _ = r.k.Finding(watchman, own, nil, true) + } + if c.end != nil { + _ = r.k.Have("novox.mesh-controller", c.end, false) + } else { + r.now = r.now.Add(delivery.DefaultDeliveryBound) + _ = r.k.BoundPassed("novox.mesh-controller") + } + if got := r.changes(); got != c.said { + t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said) + } + if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) { + t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation) + } + if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 { + t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held()) + } + } + // A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held + // while the assigner is delivered on ace, though the seat is named mesh-placement. + r := newRig() + placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}}, + Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}}) + if err != nil { + t.Fatal(err) + } + r.k.Placement("placement.ace", placed, false) + _ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now) + _ = r.k.Finding(selfCheck, seat, seatRaw, false) + if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" { + t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held()) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go new file mode 100644 index 00000000..06ad8187 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -0,0 +1,160 @@ +package main + +// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe +// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from +// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the +// lease write, the controller's conditions of the kinds that end with it. +// +// The rows are ADR 0564's and the lead's table of 2026-10-29, by the controller's source of each kind. A kind with no +// row is named by `compare` as "no home decided". + +// Home is where one condition kind lives. +type Home struct { + // Source is the controller's row or probe that raises it today, for whoever reads the table. + Source string + // Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no + // difference in compare. + Controller bool + // Observer is the module whose findings raise it after the take-over; empty for a kind that ends. + Observer string + // Carried says it ends with the controller's lease and is carried from the controller's bucket until then + // (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller). + Carried bool + // Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in + // compare and not carried. + Replaced bool + // Why is why a kind is carried when the record's table gave it another home: said by compare beside it. + Why string +} + +// Dropped is the reason of the probes the operator dropped from this release (D2, D3, D4, D12, D-heard, D-root, D11, +// and D13's data held twice): their kinds end with the controller and are carried until the lease write. +const Dropped = "dropped 2026-10-30 by the operator; the owning module declares its own check later (research 074)" + +// The kinds' homes, by kind: ADR 0564's rows, and the 63 kinds the controller's code raises that the record left without +// one, as the lead settled them under the operator's yes on 2026-10-29 (.work/47-release/kinds-section.html, with the +// lead's three changes: lease-split and image-missing do not end, and the data rows read their readings from disk-load). +var Homes = map[string]Home{ + // The watchman's watchdogs (clause 4). + "silent": {Source: "S1", Controller: true, Observer: watchman}, + "max-deliveries": {Source: "S9", Controller: true, Observer: watchman}, + "slow-consumer": {Source: "S9", Controller: true, Observer: watchman}, + "refused": {Source: "S9", Controller: true, Observer: watchman}, + "connection-closed": {Source: "S9", Controller: true, Observer: watchman}, + "bus-faults-unread": {Source: "S9", Controller: true, Observer: watchman}, + "tools-silent": {Source: "S11", Controller: true, Observer: watchman}, + "apply-failing": {Source: "S21", Controller: true, Observer: watchman}, + "module-unhealthy": {Source: "health", Controller: true, Observer: watchman}, + "stuck": {Source: "DM", Controller: true, Observer: watchman}, + "provider-failing": {Source: "S8", Controller: true, Observer: watchman}, + "provider-silent": {Source: "S8", Controller: true, Observer: watchman}, + "self-check-silent": {Source: "S10", Controller: true, Observer: watchman}, + // From the provision keys in waits (ADR 0570 §7), a setting's wait from the node's own health report. + "needs-operator": {Source: "health", Controller: true, Observer: watchman}, + // From the vault's record of each secret given (ADR 0570 §7). + "secret-given": {Source: "desk", Controller: true, Observer: watchman}, + // The watchman reading each node's own health report key (a4's, mesh-host). + "machine-network": {Source: "node", Controller: true, Observer: watchman}, + "network-rewritten": {Source: "node", Controller: true, Observer: watchman}, + "network-unreachable": {Source: "node", Controller: true, Observer: watchman}, + "machine-units": {Source: "node", Controller: true, Observer: watchman}, + "relogin-needed": {Source: "health", Controller: true, Observer: watchman}, + "directory-used-as-found": {Source: "health", Controller: true, Observer: watchman}, + // The self-check's probes (clause 6, the Context's table). + "lease-split": {Source: "D5", Controller: true, Observer: selfCheck}, // a seat held once answers from two holders + "consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck}, + "consumer-lost": {Source: "D6", Controller: true, Observer: selfCheck}, // and the watchman's S9 + "consumer-behind": {Source: "D6", Controller: true, Observer: selfCheck}, + "stream-wrong": {Source: "D7", Controller: true, Observer: selfCheck}, + "own-address-banned": {Source: "D8", Controller: true, Observer: selfCheck}, + "data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck}, + "data-shrank": {Source: "D13", Controller: true, Observer: selfCheck}, // readings from disk-load (L7) + "empty-replacement": {Source: "D13", Controller: true, Observer: selfCheck}, + "data-quiet": {Source: "D13", Controller: true, Observer: selfCheck}, + "backup-stale": {Source: "D13", Controller: true, Observer: selfCheck}, + "data-missing": {Source: "D13", Controller: true, Observer: selfCheck}, + "array-degraded": {Source: "D13", Controller: true, Observer: selfCheck}, + "protection-missing": {Source: "D13", Controller: true, Observer: selfCheck}, + "bus-reconnects": {Source: "D15", Controller: true, Observer: selfCheck}, + "bucket-or-log-filling": {Source: "fill", Controller: true, Observer: selfCheck}, + "bucket-or-log-unread": {Source: "fill", Controller: true, Observer: selfCheck}, + "watchdogs-silent": {Source: "DW", Controller: true, Observer: selfCheck}, + "core-unhealthy": {Source: "H-bus", Controller: true, Observer: selfCheck}, + "agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck}, + "probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck}, + // The healers' own (clause 7, Limits 2). + "healer-wanted": {Source: "S15", Controller: true, Observer: healers}, + "healers-braked": {Source: "healers", Controller: true, Observer: healers}, + // The assigner's, as its own findings (the bus step, ADR 0565 §9). + "bus-maintenance": {Source: "bus", Controller: true, Observer: assigner}, + "bus-upgrade-failed": {Source: "DB", Controller: true, Observer: assigner}, + // The provisioner's (ADR 0570 §5): its own ask replaces retire-waiting; a declined key stays its finding. + "retire-waiting": {Source: "retire", Controller: true, Observer: provisioner}, + "retire-rejected": {Source: "retire", Controller: true, Observer: provisioner}, + // The router's own: no channel delivered a question. + "asks-undelivered": {Source: "asks", Controller: true, Observer: messenger}, + // The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding. + "image-missing": {Source: "S25", Controller: true, Observer: artifactStore}, + // The kinds that end with the controller and are carried until its lease write (clauses 4, 9). + "controller-deaf": {Source: "S4", Controller: true, Carried: true}, + "lease-lost": {Source: "S12", Controller: true, Carried: true}, + "call-hung": {Source: "S7", Controller: true, Carried: true}, + "stale-writer": {Source: "S13", Controller: true, Carried: true}, + "facts-stale": {Source: "S14", Controller: true, Carried: true}, + "status-slow": {Source: "D9", Controller: true, Carried: true}, + "vault-silent": {Source: "send", Controller: true, Carried: true}, + "declaration-files-not-put": {Source: "send", Controller: true, Carried: true}, + "config-not-followed": {Source: "config", Controller: true, Carried: true}, + "assignment-not-made": {Source: "assign", Controller: true, Carried: true}, + "builds-not-laid-out": {Source: "start", Controller: true, Carried: true}, + "sent-builds-not-copied": {Source: "start", Controller: true, Carried: true}, + "fingerprint-not-composed": {Source: "done keys", Controller: true, Carried: true}, + "take-in-not-consumed": {Source: "take-in", Controller: true, Carried: true}, + "take-in-stalled": {Source: "take-in", Controller: true, Carried: true}, + "new-module-waits": {Source: "register", Controller: true, Carried: true}, + "left-out": {Source: "send", Controller: true, Carried: true}, + "declaration-refused": {Source: "D1", Controller: true, Carried: true}, + "unknown-field": {Source: "read", Controller: true, Carried: true}, + "bus-objects-unasserted": {Source: "send", Controller: true, Carried: true}, + // The probes the operator dropped from this release (D2 resolvers, D3 holders, D4 archives, D12 bindings, D-heard + // withheld hearings, D-root, D11 retired data, and D13's data held twice): their kinds are carried until the lease + // write. + "resolver-wrong": {Source: "D2", Controller: true, Carried: true, Why: Dropped}, + "holder-silent": {Source: "D3", Controller: true, Carried: true, Why: Dropped}, + "archives-missing": {Source: "D4", Controller: true, Carried: true, Why: Dropped}, + "archives-unheld": {Source: "D4", Controller: true, Carried: true, Why: Dropped}, + "binding-kept": {Source: "D12", Controller: true, Carried: true, Why: Dropped}, + "binding-moved": {Source: "D12", Controller: true, Carried: true, Why: Dropped}, + "binding-moving": {Source: "D12", Controller: true, Carried: true, Why: Dropped}, + "grant-composes-nothing": {Source: "withheld", Controller: true, Carried: true, Why: Dropped}, + "root-not-free": {Source: "D-root", Controller: true, Carried: true, Why: Dropped}, + "cleanup-waiting": {Source: "D11", Controller: true, Carried: true, Why: Dropped}, + "data-held-twice": {Source: "D13", Controller: true, Carried: true, Why: Dropped}, + // The lead, 2026-10-29: no delivery plans since R6 (wants replace them), and a stalled delivery is the watchman's + // stuck finding seen directly. + "plan-failed": {Source: "S23", Controller: true, Carried: true}, + "stalled": {Source: "D14", Controller: true, Carried: true}, + // The kinds that end, their faults observed otherwise (the Context's table). + "core-behind": {Source: "D10", Controller: true, Replaced: true}, +} + +// The observers, as the table names them. +const ( + watchman = "mesh-delivery-watchman" + selfCheck = "mesh-self-check" + healers = "mesh-healers" + assigner = "mesh-delivery-assigner" + provisioner = "mesh-provisioner" + messenger = "messenger" + artifactStore = "artifact-store-tools" +) + +// homeOf is the home of a condition's kind, and whether the table has one. +func (k *Keeper) homeOf(kind string) (Home, bool) { + homes := k.Homes + if homes == nil { + homes = Homes + } + h, ok := homes[kind] + return h, ok +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go new file mode 100644 index 00000000..a167631c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -0,0 +1,668 @@ +package main + +// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the +// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and, +// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From +// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set, +// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition +// but its finding going (ADR 0231 §14). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The module, its seat, and the states it keeps. +const ( + Module = health.KeeperModule + Seat = "mesh-conditions" + + ConditionsState = health.ConditionsState + TakeoverState = health.TakeoverState + HistoryState = "history" + HeldState = "held" + BoundsState = "bounds" + // AskedState keeps the keeper's asks of the operator (go/asker). + AskedState = "asked" +) + +// HistoryKeptFor is how long a transition is kept (clause 3). +const HistoryKeptFor = 90 * 24 * time.Hour + +// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's; the lead's +// table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder, +// which observes S25, image-missing). +var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule, + "mesh-provisioner", "messenger", artifactStore} + +// The other states it reads, as `.`. +const ( + AttemptsRead = health.HealersModule + "." + health.AttemptsState + ControllerConditions = "mesh-controller.conditions" + ControllerLease = "mesh-controller.lease" + ControllerLeaseKey = "holder" + WantsRead = delivery.AssignerModule + ".wants" + // PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the + // lead, 2026-10-29, for ADR 0564 clause 2's inputs). + PlacementRead = delivery.PlacementRead + HavesRead = "node-engine.have" + ConditionWrites = 8 +) + +// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to +// live for a hold's bound. +type Store interface { + Get(state, key string) (value []byte, revision uint64, found bool, err error) + Create(state, key string, value any) (uint64, error) + Update(state, key string, value any, revision uint64) (uint64, error) + Delete(state, key string) error + Keys(state string) ([]string, error) + CreateTTL(state, key string, value any, ttl time.Duration) error +} + +// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)). +type held struct { + Key string `json:"key"` + Node string `json:"node"` + Module string `json:"module"` + Since time.Time `json:"since"` + Version string `json:"version"` + Wanted json.RawMessage `json:"-"` +} + +// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening. +type clearedMemo struct { + at time.Time + count int + began time.Time + gaps []health.Gap + silenced *health.Silence + tried []health.Tried +} + +// Keeper is the condition keeper's state and its writes. +type Keeper struct { + Store Store + // Say publishes one event on the seat; History keeps one transition. + Say func(e health.Event) error + Now func() time.Time + Logf func(format string, a ...any) + // Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil. + Homes map[string]Home + // Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go). + Bound time.Duration + + mu sync.Mutex + findings map[string]map[string]health.Finding + blind map[string]string + attempts map[string]health.Attempt + ctrl map[string]health.Condition + leaseOut bool + takeover health.Takeover + wants map[string]delivery.Want + haves map[string]delivery.Have + // holders are, by `.`, the module the placement says holds the seat there. + holders map[string]string + stuck map[string]string + held map[string]held + // passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the + // line a key raised out of a hold says (hold.go). + passed map[string]string + outOfHold map[string]string + cleared map[string]clearedMemo + seq uint64 +} + +func (k *Keeper) init() { + if k.findings == nil { + k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{}, + map[string]health.Attempt{}, map[string]health.Condition{} + k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{}, + map[string]string{}, map[string]held{}, map[string]clearedMemo{} + k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController} + k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{} + } + if k.Now == nil { + k.Now = time.Now + } + if k.Logf == nil { + k.Logf = func(string, ...any) {} + } +} + +// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears +// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was. +func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if k.findings[observer] == nil { + k.findings[observer] = map[string]health.Finding{} + } + if deleted { + delete(k.findings[observer], key) + } else { + f, err := health.DecodeFinding(value) + if err != nil { + k.mu.Unlock() + k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err) + return nil + } + if f.Key != key { + k.mu.Unlock() + k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key) + return nil + } + k.findings[observer][key] = f + } + k.mu.Unlock() + return k.Settle(key) +} + +// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing +// (ADR 0227 §4). Empty why says it is read again. +func (k *Keeper) Blind(observer, why string) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + if why == "" { + delete(k.blind, observer) + } else { + k.blind[observer] = why + } +} + +// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation +// (clause 7). +func (k *Keeper) Attempt(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + var condition string + if old, ok := k.attempts[key]; ok { + condition = old.Condition + } + if deleted { + delete(k.attempts, key) + } else { + var a health.Attempt + if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" { + k.mu.Unlock() + k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err) + return nil + } + k.attempts[key], condition = a, a.Condition + } + k.mu.Unlock() + if condition == "" { + return nil + } + return k.Settle(condition) +} + +// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write. +func (k *Keeper) Controller(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if deleted { + delete(k.ctrl, key) + } else { + var c health.Condition + if err := json.Unmarshal(value, &c); err != nil { + k.mu.Unlock() + k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err) + return nil + } + k.ctrl[key] = c + } + k.mu.Unlock() + return k.Settle(key) +} + +// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no +// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear. +func (k *Keeper) Lease(away bool) error { + k.mu.Lock() + k.init() + was := k.leaseOut + k.leaseOut = away + keys := make([]string, 0, len(k.ctrl)) + for key := range k.ctrl { + keys = append(keys, key) + } + k.mu.Unlock() + if was == away { + return nil + } + return k.settleAll(keys) +} + +// LeaseAway says whether the controller's lease is handed away. +func (k *Keeper) LeaseAway() bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.leaseOut +} + +// Takeover takes the keeper's own take-over key as read. +func (k *Keeper) Takeover(t health.Takeover) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + k.takeover = t +} + +// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves +// it under this one. +func (k *Keeper) Settle(key string) error { + var err error + for i := 0; i < ConditionWrites; i++ { + if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err) +} + +func (k *Keeper) settleAll(keys []string) error { + sort.Strings(keys) + var problems []string + for _, key := range keys { + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's +// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it. +func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) { + observers := make([]string, 0, len(k.findings)) + for o := range k.findings { + observers = append(observers, o) + } + sort.Strings(observers) + for _, o := range observers { + if f, ok := k.findings[o][key]; ok { + return &f, nil + } + } + if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) { + return nil, &c + } + return nil, nil +} + +func (k *Keeper) endsWithController(c health.Condition) bool { + h, ok := k.homeOf(c.Kind) + return ok && h.Carried +} + +// settleOnce is one try of Settle. +func (k *Keeper) settleOnce(key string) error { + now := k.Now().UTC() + raw, rev, open, err := k.Store.Get(ConditionsState, key) + if err != nil { + return fmt.Errorf("reading the condition %s: %w", key, err) + } + var was health.Condition + if open { + if err := json.Unmarshal(raw, &was); err != nil { + return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + } + k.mu.Lock() + k.init() + f, carried := k.wanted(key) + hold, holding := k.holdFor(key, f, carried, open) + k.mu.Unlock() + + if f == nil && carried == nil { + k.dropHeld(key, "its finding went while its module was delivered") + if !open { + return nil + } + return k.clear(key, was, rev, now, "its finding went") + } + if holding { + return k.keepHeld(hold) + } + k.dropHeld(key, "") + next := k.derive(key, was, open, f, carried, now) + events := transitions(was, next, open) + if len(events) == 0 && open && sameBody(was, next) { + return nil + } + if open { + _, err = k.Store.Update(ConditionsState, key, next, rev) + } else { + _, err = k.Store.Create(ConditionsState, key, next) + } + if err != nil { + return err + } + k.mu.Lock() + delete(k.cleared, key) + k.mu.Unlock() + for _, e := range events { + e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change) + e.Show = k.show(next, carried != nil) + k.tell(e) + } + return nil +} + +func (k *Keeper) show(c health.Condition, carried bool) string { + if carried { + return health.ShowController(c.Key) + } + return health.ShowKeeper(c.Key) +} + +// derive is the condition key should be now, from what was open and what the inputs say. +func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition { + var c health.Condition + switch { + case carried != nil: + // Raised and cleared as the controller's bucket says, with its show and actions (Limits 9). + c = *carried + c.Epoch = 0 + c.LooksUnread = "" + case open: + c = was + k.fromFinding(&c, *f, now) + default: + c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf} + k.fromFinding(&c, *f, now) + k.mu.Lock() + if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin { + c.Count = before.count + 1 + c.First = before.began + c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now}) + if over := len(c.Gaps) - health.KeptGaps; over > 0 { + c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:] + } + if before.silenced != nil && now.Before(before.silenced.Until) { + c.Silenced = before.silenced + } + c.Tried = before.tried + } + k.mu.Unlock() + } + if carried == nil { + k.overlay(&c, now) + } + if !open { + k.mu.Lock() + if line := k.outOfHold[key]; line != "" { + c.Explanation = strings.TrimSpace(c.Explanation + " " + line) + } + k.mu.Unlock() + } + if c.Silenced != nil && !now.Before(c.Silenced.Until) { + c.Silenced = nil + } + return c +} + +// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence +// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again. +func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) { + escalated := c.Escalated() + c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source + c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions + if !(escalated && f.Severity != health.Urgent) { + c.Severity = f.Severity + } + if !escalated { + c.Resolver = f.Resolver + if c.Resolver == "" { + c.Resolver = health.ResolverSelf + } + } + if len(f.Evidence) > 0 { + c.Evidence = append([]health.Evidence(nil), f.Evidence...) + c.LastObserved = f.Evidence[0].At + } else if c.Evidence == nil { + c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}} + c.LastObserved = f.At + } + c.Observations = len(c.Evidence) + if c.Actions == nil { + c.Actions = []health.Action{} + } + _ = now +} + +// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's +// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has +// none of its own. +func (k *Keeper) overlay(c *health.Condition, now time.Time) { + k.mu.Lock() + var mine []health.Attempt + for _, a := range k.attempts { + if a.Condition == c.Key { + mine = append(mine, a) + } + } + ctrl, carry := k.ctrl[c.Key] + carry = carry && !k.leaseOut + k.mu.Unlock() + sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) }) + if carry && len(ctrl.Tried) > 0 { + c.Tried = append([]health.Tried(nil), ctrl.Tried...) + if ctrl.Escalated() { + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + } else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf { + c.Resolver = ctrl.Resolver + } + } + for _, a := range mine { + t := a.Tried() + if !containsTried(c.Tried, t) { + c.Tried = append(c.Tried, t) + } + switch a.Outcome { + case health.OutcomeEscalated: + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + default: + if c.Resolver != health.ResolverOperator { + c.Resolver = health.ResolverHealer(a.Healer) + } + } + } + if len(c.Tried) > health.KeptTried { + c.Tried = c.Tried[len(c.Tried)-health.KeptTried:] + } + if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) { + c.Silenced = ctrl.Silenced + } +} + +func containsTried(list []health.Tried, t health.Tried) bool { + for _, x := range list { + if x.At.Equal(t.At) && x.By == t.By && x.What == t.What { + return true + } + } + return false +} + +// transitions are the events between what was open and what is now, in the controller's order: raised or reopened; +// else severity, resolver, silence and words changes. An observation that changes none of these says nothing. +func transitions(was, now health.Condition, open bool) []health.Event { + if !open { + change := health.ChangeRaised + if now.Count > 1 { + change = health.ChangeReopened + } + return []health.Event{{Change: change}} + } + var out []health.Event + if was.Severity != now.Severity { + out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)}) + } + if was.Resolver != now.Resolver { + out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver}) + } + switch { + case was.Silenced == nil && now.Silenced != nil: + out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why}) + case was.Silenced != nil && now.Silenced == nil: + out = append(out, health.Event{Change: health.ChangeUnsilenced}) + } + if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) { + out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs}) + } + return out +} + +func sameActions(a, b []health.Action) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) || (len(a) == 0 && len(b) == 0) +} + +func sameBody(a, b health.Condition) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// clear removes an open condition whose finding went, and says it (clause 2). +func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error { + // A delete at the revision read: a write that moved it meanwhile is read again. + if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil { + return err + } + if err := k.Store.Delete(ConditionsState, key); err != nil { + return err + } + k.mu.Lock() + k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried} + k.mu.Unlock() + cleared := now + k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why, + Cleared: &cleared, Show: k.show(was, false)}) + return nil +} + +// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition +// and its history still hold it. +func (k *Keeper) tell(e health.Event) { + k.mu.Lock() + k.seq++ + hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq) + k.mu.Unlock() + if _, err := k.Store.Create(HistoryState, hk, e); err != nil { + k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err) + } + if k.Say == nil { + return + } + if err := k.Say(e); err != nil { + k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err) + } +} + +// Open is every open condition, ordered as `conditions` says them. +func (k *Keeper) Open() ([]health.Condition, error) { + keys, err := k.Store.Keys(ConditionsState) + if err != nil { + return nil, err + } + var out []health.Condition + for _, key := range keys { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + continue + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + out = append(out, c) + } + health.Order(out) + return out, nil +} + +// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared. +func (k *Keeper) Resync() (int, error) { + open, err := k.Open() + if err != nil { + return 0, err + } + seen := map[string]bool{} + for _, c := range open { + seen[c.Key] = true + } + k.mu.Lock() + k.init() + for o, fs := range k.findings { + if k.blind[o] != "" { + continue + } + for key := range fs { + seen[key] = true + } + } + for key := range k.ctrl { + seen[key] = true + } + k.mu.Unlock() + keys := make([]string, 0, len(seen)) + for key := range seen { + keys = append(keys, key) + } + return len(keys), k.settleAll(keys) +} + +// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in +// history, naming who and why. +func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) { + if strings.TrimSpace(why) == "" { + return health.Condition{}, errors.New("a silence says why") + } + if d <= 0 || d > health.MaxSilence { + return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d) + } + for i := 0; i < ConditionWrites; i++ { + raw, rev, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return health.Condition{}, err + } + if !found || string(raw) == "null" { + return health.Condition{}, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return health.Condition{}, err + } + now := k.Now().UTC() + c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now} + if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) { + continue + } else if err != nil { + return health.Condition{}, err + } + k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced, + Why: why + " (" + by + ")", Show: health.ShowKeeper(key)}) + return c, nil + } + return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key) +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go new file mode 100644 index 00000000..475f8ec5 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -0,0 +1,341 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// fakeStore is the bus's states in memory, with revisions, as the runtime answers them. +type fakeStore struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + next uint64 + // ttls are the times to live the keys were created with. + ttls map[string]time.Duration +} + +func newFakeStore() *fakeStore { + return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}} +} + +func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.vals[state+"/"+key] + return v, f.revs[state+"/"+key], ok, nil +} + +func (f *fakeStore) put(state, key string, value any) uint64 { + raw, _ := json.Marshal(value) + f.next++ + f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next + return f.next +} + +func (f *fakeStore) Create(state, key string, value any) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.vals[state+"/"+key]; ok { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.revs[state+"/"+key] != rev { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Delete(state, key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.vals, state+"/"+key) + delete(f.revs, state+"/"+key) + return nil +} + +func (f *fakeStore) Keys(state string) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.vals { + if s, key, _ := strings.Cut(k, "/"); s == state { + out = append(out, key) + } + } + sort.Strings(out) + return out, nil +} + +func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + f.mu.Lock() + if f.ttls == nil { + f.ttls = map[string]time.Duration{} + } + f.ttls[state+"/"+key] = ttl + f.mu.Unlock() + _, err := f.Create(state, key, value) + return err +} + +// rig is a keeper over a fake store, its clock and what it said. +type rig struct { + k *Keeper + st *fakeStore + now time.Time + said []health.Event +} + +func newRig() *rig { + r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)} + r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error { + r.said = append(r.said, e) + return nil + }} + return r +} + +func (r *rig) changes() string { + var out []string + for _, e := range r.said { + out = append(out, e.Change+" "+e.Key) + } + r.said = nil + return strings.Join(out, ", ") +} + +func (r *rig) open(key string) (health.Condition, bool) { + raw, _, ok, _ := r.st.Get(ConditionsState, key) + var c health.Condition + if ok { + _ = json.Unmarshal(raw, &c) + } + return c, ok +} + +// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding). +func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) { + t.Helper() + f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id}, + Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source, + Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + return f.Key, raw +} + +// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer +// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten +// minutes is the same fault; a silence is a transition in history. +func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) { + r := newRig() + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + steps := []struct { + name string + do func() error + said string + check func(c health.Condition, open bool) string + }{ + {name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) }, + said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string { + if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }}, + {name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err }, + check: func(c health.Condition, open bool) string { + if !open { + return "cleared by blindness" + } + return "" + }}, + {name: "a healer's attempt reaches tried and the resolver", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now}) + return r.k.Attempt("h3-1", a, false) + }, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "an escalation makes it urgent, the operator's", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated, + Begun: r.now.Add(time.Minute)}) + return r.k.Attempt("h3-2", a, false) + }, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "a silence is a transition with who and why", do: func() error { + _, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable") + return err + }, said: "silenced machine.g14.silent"}, + {name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) }, + said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string { + if open { + return "still open" + } + return "" + }}, + {name: "raised again within ten minutes is a reopening", do: func() error { + r.now = r.now.Add(5 * time.Minute) + return r.k.Finding(watchman, key, raw, false) + }, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Count != 2 || len(c.Gaps) != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + // Until the lease write the controller's escalation for the same key reaches the condition (clause 8). + {name: "the controller's escalation is carried", do: func() error { + ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator, + Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}}) + _ = r.k.Attempt("h3-1", nil, true) + _ = r.k.Attempt("h3-2", nil, true) + return r.k.Controller(key, ctrl, false) + }, check: func(c health.Condition, _ bool) string { + if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + } + for _, s := range steps { + if err := s.do(); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" { + t.Errorf("%s: said %q, want %q", s.name, got, s.said) + } + if s.check != nil { + c, open := r.open(key) + if why := s.check(c, open); why != "" { + t.Errorf("%s: %s", s.name, why) + } + } + } + // Every transition is kept in history, and each event carries the keeper's show. + keys, _ := r.st.Keys(HistoryState) + if len(keys) < 7 { + t.Errorf("history keeps %d transitions", len(keys)) + } + raw0, _, _, _ := r.st.Get(HistoryState, keys[0]) + if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) { + t.Errorf("the first transition is %s", raw0) + } + // A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the + // lease is handed away, with the controller's show (clause 9, Limits 9). + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9}) + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" { + t.Errorf("a carried kind is %+v, said %+v", c, r.said) + } + r.said = nil + if err := r.k.Lease(true); err != nil { + t.Fatal(err) + } + if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" { + t.Error("a carried kind outlived the lease write") + } +} + +// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held +// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed +// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and +// hand-back after the lease write. +func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) { + r := newRig() + put := func(key, kind string) []byte { + raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"}) + return raw + } + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + _ = r.k.Finding(watchman, key, raw, false) + _ = r.k.Controller(key, put(key, "silent"), false) + only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now) + _ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false) + _ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false) + _ = r.k.Controller("delivery.x.mystery", put("delivery.x.mystery", "mystery"), false) + _ = r.k.Controller("bus.x.consumer-wrong", put("bus.x.consumer-wrong", "consumer-wrong"), false) + _ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false) + cmp, err := r.k.Compare() + if err != nil { + t.Fatal(err) + } + reasons := map[string]string{} + for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) { + reasons[d.Key] = d.Reason + } + for key, says := range map[string]string{ + only: "keeper only", + "core.controller.deaf": "carried", + "delivery.x.mystery": "no home is decided for the kind mystery", + "bus.x.consumer-wrong": "mesh-self-check has not written its finding", + // A kind of a probe the operator dropped (2026-10-30) is carried, never missing. + "seat.vault.ace.holder-silent": "carried from its bucket until the lease write (dropped 2026-10-30 by the operator", + } { + if !strings.Contains(reasons[key], says) { + t.Errorf("%s: %q, want it to say %q", key, reasons[key], says) + } + } + if cmp.Same != 1 || len(cmp.Differences) != 2 { + t.Errorf("compare: %+v", cmp) + } + // The lead's table of 2026-10-29: each kind has its home, lease-split and image-missing do not end, and the 22 + // that end are carried; with the 11 kinds of the probes the operator dropped on 2026-10-30, 33. + carried := 0 + for _, h := range Homes { + if h.Carried { + carried++ + } + } + for kind, observer := range map[string]string{"needs-operator": watchman, "machine-network": watchman, "lease-split": selfCheck, + "data-shrank": selfCheck, "bus-upgrade-failed": assigner, "retire-waiting": provisioner, "asks-undelivered": messenger, + "image-missing": artifactStore} { + if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried { + t.Errorf("%s's home is %+v", kind, h) + } + } + for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept", + "binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} { + if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != Dropped { + t.Errorf("the dropped probe's kind %s is %+v", kind, h) + } + } + if len(Homes) != 84 || carried != 33 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled + t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried) + } + if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") { + t.Errorf("take-over while compare differs: %v", err) + } + if _, err := r.k.HandBack("jochen", "back"); err != nil { + t.Errorf("hand-back before the lease write: %v", err) + } + _ = r.k.Lease(true) + if _, err := r.k.HandBack("jochen", "back"); err == nil { + t.Error("hand-back after the lease write was taken") + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go new file mode 100644 index 00000000..ac597d4b --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -0,0 +1,164 @@ +// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A +// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its +// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the +// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's +// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the +// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP +// channel; what this module says, it says on stderr. +package main + +import ( + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} + +func (runtimeStore) Create(state, key string, value any) (uint64, error) { + return stdio.State(state).Create(key, value) +} + +func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) { + return stdio.State(state).Update(key, value, revision) +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire. +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) + return err +} + +// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3). +func sayOnSeat(e health.Event) error { + return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "") +} + +// NewKeeper is the keeper over the runtime. +func NewKeeper() *Keeper { + return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf} +} + +// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not +// delivered) is read again when it is. +func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) { + go func() { + for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) { + err := stdio.State(state).Watch("", handle) + if err == nil { + if observer != "" { + k.Blind(observer, "") + } + return + } + if observer != "" { + k.Blind(observer, err.Error()) + } + logf("%s cannot be watched (%v); watched again in %s", state, err, wait) + time.Sleep(wait) + } + }() +} + +// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13). +func Keep(k *Keeper) { + for _, s := range []string{HeldState} { + keys, _ := stdio.State(s).Keys() + for _, key := range keys { + if e, err := stdio.State(s).Get(key); err == nil && e != nil { + k.restoreHeld(key, e.Value) + } + } + } + if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil { + var raw []byte + if e != nil { + raw = e.Value + } + if t, err := health.DecodeTakeover(raw, e != nil); err == nil { + k.Takeover(t) + } + } + said := func(err error) error { + if err != nil { + logf("%v", err) + } + return nil + } + watch(k, ControllerLease, "", func(c stdio.StateChange) error { + if c.Key != ControllerLeaseKey { + return nil + } + away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil) + logf("the controller's lease: %s", why) + return said(k.Lease(away)) + }) + watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) }) + watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) }) + watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error { + return said(k.Attempt(c.Key, c.Value, c.Op != "put")) + }) + watch(k, ControllerConditions, "", func(c stdio.StateChange) error { + if k.LeaseAway() { + return nil + } + return said(k.Controller(c.Key, c.Value, c.Op != "put")) + }) + for _, o := range Observers { + watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error { + return said(k.Finding(o, c.Key, c.Value, c.Op != "put")) + }) + } + watch(k, BoundsState, "", func(c stdio.StateChange) error { + if c.Op == stdio.OpExpire { + return said(k.BoundPassed(c.Key)) + } + return nil + }) + watch(k, PlacementRead, "", func(c stdio.StateChange) error { + k.Placement(c.Key, c.Value, c.Op != stdio.OpPut) + return nil + }) +} + +func main() { + k := NewKeeper() + client := asker.New(AskedState) + client.Module = Module + go func() { + <-stdio.Served() + Keep(k) + if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) { + if err != nil { + logf("the answer to %s (%s): %v", w.Ask, h, err) + } + }); err != nil { + logf("the operator's answers cannot be heard: %v", err) + } + }() + if err := stdio.Serve("", Tools(k, client)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go new file mode 100644 index 00000000..94b1801c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat +// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers' +// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its +// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it, +// every act a tool that starts it. +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("the manifest is refused: %v", err) + } + seat, _ := catalogue.SeatNamed(Seat) + if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat || + !slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) { + t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves)) + } + reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead} + for _, o := range Observers { + reads = append(reads, o+"."+health.FindingsState) + } + slices.Sort(reads) + got := slices.Clone(m.Reads) + slices.Sort(got) + if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) { + t.Errorf("reads %v, want %v and never the healers' would", got, reads) + } + ttl, keyTTL := map[string]int{}, map[string]bool{} + for _, s := range m.State { + ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL + } + // Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age. + if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] { + t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v", + ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState]) + } + // Clause 12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool, + HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync", + "what it reads": InputsTool} + tools := map[string]bool{} + for _, tool := range Tools(&Keeper{}, nil) { + tools[tool.Name] = true + } + for name := range ttl { + if readBy[name] == "" { + t.Errorf("the state %s has no tool that reads it", name) + } + } + for what, tool := range readBy { + if !tools[tool] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/reads-outside b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside new file mode 100644 index 00000000..c20a67a5 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-conditions/module.json diff --git a/modules/mesh-conditions/cmd/mesh-conditions/tools.go b/modules/mesh-conditions/cmd/mesh-conditions/tools.go new file mode 100644 index 00000000..330e153e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/tools.go @@ -0,0 +1,252 @@ +package main + +// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions` +// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the +// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call. + +import ( + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"} + +// The keeper's own read tools. +const ( + HistoryTool = "conditions_history" + HeldTool = "conditions_held" + TakeoverTool = "conditions_takeover" + InputsTool = "conditions_inputs" +) + +// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314). +const briefEvidence = 1 + +// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole. +type inBrief struct { + health.Condition + EvidenceKept int `json:"evidence-kept,omitempty"` + More string `json:"more,omitempty"` +} + +func obj(props map[string]any, required ...string) map[string]any { + out := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + out["required"] = required + } + return out +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +// Tools are what the runtime serves. +func Tools(k *Keeper, a Asks) []stdio.Tool { + return []stdio.Tool{ + {Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " + + "oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " + + "one whole; with history, every transition lately.", + Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"), + "severity": str("urgent or warning"), "machine": str("only those about this machine"), + "history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}), + Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }}, + {Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " + + "transition in history; it stays open.", + Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"), + "why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"), + Run: func(args map[string]any) (any, error) { + d, err := duration(strArg(args, "for")) + if err != nil { + return nil, err + } + return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " + + "its reason; what ends with the controller, what only the keeper raises and what is held listed apart.", + Run: func(map[string]any) (any, error) { return k.Compare() }}, + {Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " + + "keeper; refused while compare differs.", + Input: obj(map[string]any{"why": str("why now")}, "why"), + Run: func(args map[string]any) (any, error) { + return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " + + "is handed away.", + Input: obj(map[string]any{"why": str("why")}, "why"), + Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }}, + {Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.", + Run: func(map[string]any) (any, error) { + n, err := k.Resync() + return map[string]any{"settled": n}, err + }}, + {Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.", + Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}), + Run: func(args map[string]any) (any, error) { + return k.History(strArg(args, "key"), intArg(args, "limit", 50)) + }}, + {Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " + + "with its module, node, version and since when, and the deliveries' bounds.", + Run: func(map[string]any) (any, error) { + bounds, err := k.Store.Keys(BoundsState) + return map[string]any{"held": k.Held(), "bounds": bounds}, err + }}, + {Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.", + Run: func(map[string]any) (any, error) { + raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return nil, err + } + return health.DecodeTakeover(raw, found) + }}, + {Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " + + "cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.", + Run: func(map[string]any) (any, error) { return k.Inputs(), nil }}, + } +} + +// ConditionsVerb is `conditions`: the list, one whole, or the history. +func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) { + key := strArg(args, "key") + if strArg(args, "history") == "true" { + days := intArg(args, "days", 7) + if days < 1 || days > 90 { + return nil, fmt.Errorf("days is from 1 to 90, not %d", days) + } + return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days) + } + if key != "" { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + return nil, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + return c, json.Unmarshal(raw, &c) + } + scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine") + if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) { + return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity) + } + open, err := k.Open() + if err != nil { + return nil, err + } + list := []inBrief{} + counted := map[string]int{} + for _, c := range open { + if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) || + (machine != "" && !concerns(c, machine)) { + continue + } + b := inBrief{Condition: c} + if len(c.Evidence) > briefEvidence { + b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key) + } + list = append(list, b) + counted[c.Source+" "+c.Kind]++ + } + return map[string]any{"conditions": list, "open": len(open), "counted": counted, + "note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " + + "its newest evidence — `conditions key=` gives one whole"}, nil +} + +// concerns says whether a condition is about a machine: it names it, or its key does. +func concerns(c health.Condition, machine string) bool { + return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || + slices.Contains(strings.Split(c.Key, "."), machine) +} + +// History is the newest transitions, at most limit, of one key or all. +func (k *Keeper) History(key string, limit int) ([]health.Event, error) { + all, err := k.HistorySince(key, time.Time{}, 0) + if err != nil { + return nil, err + } + events := all["history"].([]health.Event) + slices.Reverse(events) + if len(events) > limit { + events = events[:limit] + } + return events, nil +} + +// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it. +func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) { + keys, err := k.Store.Keys(HistoryState) + if err != nil { + return nil, err + } + events := []health.Event{} + for _, hk := range keys { + raw, _, found, err := k.Store.Get(HistoryState, hk) + if err != nil || !found { + continue + } + var e health.Event + if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) { + continue + } + events = append(events, e) + } + return map[string]any{"history": events, "days": days}, nil +} + +// Inputs is what the keeper reads, as it stands. +func (k *Keeper) Inputs() map[string]any { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + findings := map[string]int{} + for o, fs := range k.findings { + findings[o] = len(fs) + } + return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts), + "controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow, + "wants": len(k.wants), "haves": len(k.haves)} +} + +func strArg(a map[string]any, name string) string { + switch v := a[name].(type) { + case string: + return strings.TrimSpace(v) + case bool: + return strconv.FormatBool(v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + } + return "" +} + +func intArg(a map[string]any, name string, def int) int { + if n, err := strconv.Atoi(strArg(a, name)); err == nil { + return n + } + return def +} + +// duration reads 30m, 4h or 2d. +func duration(s string) (time.Duration, error) { + if n, ok := strings.CutSuffix(s, "d"); ok { + days, err := strconv.Atoi(n) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return time.Duration(days) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return d, nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go new file mode 100644 index 00000000..f8881730 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -0,0 +1,335 @@ +//go:build integration + +package main + +// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary, +// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a +// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its +// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as +// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as +// condition-cleared. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the keeper needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is a state's bucket: the keeper's own by its local name, another's by `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, read := strings.Cut(state, "."); read { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + kv, kvErr := p.kv(a.State) + switch m.Method { + case "mesh/state.get": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.create", "mesh/state.update", "mesh/state.put": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + var rev uint64 + var err error + switch m.Method { + case "mesh/state.create": + rev, err = kv.Create(p.ctx, a.Key, a.Value) + case "mesh/state.update": + rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision) + default: + rev, err = kv.Put(p.ctx, a.Key, a.Value) + } + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}}) + return + } + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.keys": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + err := kvErr + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watchID string + _ = json.Unmarshal(m.ID, &watchID) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID, + "change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())), + "revision": e.Revision(), "current": current}}}) + } + case "mesh/seat.publish": + p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body)) + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func nullIfEmpty(b []byte) []byte { + if len(b) == 0 { + return []byte("null") + } + return b +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears +// it. +func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the keeper does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement", + "mesh-healers_attempts"} + for _, o := range Observers { + buckets = append(buckets, o+"_findings") + } + for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} { + buckets = append(buckets, Module+"_"+s) + } + for _, b := range buckets { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>") + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + // The first call serves; the watches start after it. + if _, err := p.call(InputsTool, map[string]any{}); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + at := time.Now().UTC() + f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14", + Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline", + Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1", + Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings") + if _, err := findings.Put(ctx, f.Key, raw); err != nil { + t.Fatal(err) + } + msg, err := events.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("no condition-raised was said: %v", err) + } + var e health.Event + if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" || + e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline { + t.Fatalf("said %s on %s", msg.Data, msg.Subject) + } + listed, err := p.call(Seat+".conditions", map[string]any{}) + if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) { + t.Fatalf("conditions answered %s, %v", listed, err) + } + if err := findings.Delete(ctx, f.Key); err != nil { + t.Fatal(err) + } + msg, err = events.NextMsg(10 * time.Second) + if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" { + t.Fatalf("no condition-cleared was said: %v", err) + } + conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState) + if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the cleared condition is still kept: %v", err) + } +} diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod new file mode 100644 index 00000000..b941eeac --- /dev/null +++ b/modules/mesh-conditions/go.mod @@ -0,0 +1,22 @@ +module mesh-conditions + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-conditions/go.sum b/modules/mesh-conditions/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-conditions/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json new file mode 100644 index 00000000..d676b3cf --- /dev/null +++ b/modules/mesh-conditions/module.json @@ -0,0 +1,87 @@ +{ + "module": "mesh-conditions", + "version": "1", + "runs-as": "mesh-conditions", + "claims": [ + { + "name": "mesh-conditions", + "scope": "mesh", + "serves": [ + "conditions", + "silence", + "compare", + "take-over", + "hand-back", + "resync" + ] + } + ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-watchman.findings", + "mesh-self-check.findings", + "mesh-healers.findings", + "mesh-delivery-assigner.findings", + "messenger.findings", + "mesh-provisioner.findings", + "artifact-store-tools.findings", + "mesh-healers.attempts", + "mesh-controller.conditions", + "mesh-controller.lease", + "mesh-delivery-assigner.wants", + "node-engine.have", + "mesh-delivery-assigner.placement" + ], + "state": [ + "conditions", + { + "name": "history", + "ttl-seconds": 7776000 + }, + "takeover", + "held", + { + "name": "bounds", + "key-ttl": true + }, + "asked" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-conditions", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-conditions", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-conditions" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-conditions", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-conditions", + "binary": "mesh-conditions", + "loads": [ + "mesh-conditions" + ] + } + ] + } +} diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md index 42bd1ed0..72220615 100644 --- a/modules/mesh-config/README.md +++ b/modules/mesh-config/README.md @@ -61,3 +61,21 @@ them, newest first; with `proposal`, one read whole, with whether the router's a Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## The switch from the controller's records (novox/hq ADR 0565 §3, §4, §13) + +- `mesh-config_declared`: each placed module's declarations at the trunk, with the manifest and commit they come + from, written only when they differ. +- `mesh-config_takeover`: the key `switch` names the source that holds: none (the interim since mesh-catalog #558: + declarations from the modules' own manifests, the list of predefined keys and pull requests' config from the + controller), `indexer` after the operator's take-over (the list and pull requests' config from the indexer's + `predefined` and `config-candidates`; the controller's `pull-declares` refused), `controller` after a hand-back + (declarations, list and pull requests' config from the controller's records). Each take-over ask is kept beside it. +- `config_compare` lists every difference between the own sources and the controller's records: each module's + declarations, the list, each open pull request's config, and every placed module's value on every node resolved + both ways. `config_take_over` asks the operator's Approve, refused while any difference stands and switched on the + Approve only if none stands then. `config_hand_back` returns to the controller's records until its lease is handed + away. `config_declared` and `config_resync` read and write again what it keeps. +- From the controller's lease write a trusted value is set only through propose and the operator's Approve. +- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the + controller's while that seat has no holder and the controller holds its lease. diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index 0f2435cc..1928a3df 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level, strings.Join(problems, "; ")) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+ "the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+ "propose it instead, with config's action propose", call.Key, call.Caller) @@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { if !had { return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+ "operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+ "config's action propose and clear", call.Key, call.Caller) @@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return answer, nil } +// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is +// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever +// calls (novox/hq ADR 0565 §13). +func (c *Config) atTheTerminal(call Call) bool { + return call.Caller == ControllerCaller && !c.leaseAway() +} + // declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared // keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10). func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) { diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 036acd86..c0bba92e 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -19,6 +19,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. @@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) { func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) } -// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb. +// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat +// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11). +var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool { + e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +}} + +// HandAct records a change in the hand-act log (to-be 45 §7). func (runtimeStore) HandAct(what, why, cause string) error { - _, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause}) + _, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause}) return err } @@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error { func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) } func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() } func (runtimeAsks) Warranted(ask string) error { - _, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask}) + _, err := recorder.RecordWarrant(Module, ask) return err } @@ -118,7 +132,8 @@ func resolving(c *Config) { } // A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is // every change of what a module declares where (issue 1953); the builds themselves are read, not watched. - for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} { + for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead, + CandidatesRead, MirrorDeclaredRead, TakeoverState} { go func(read string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { time.Sleep(wait) @@ -221,9 +236,9 @@ func hearing(c *Config) { } } -// Tools is the seat's verb as the runtime serves it. +// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them. func Tools(c *Config) []stdio.Tool { - return []stdio.Tool{{ + return append(switchTools(c), stdio.Tool{ Name: Seat + "." + Verb, Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + @@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool { call.Caller = stdio.CallerOf(args) return c.Do(call) }, - }} + }) } // callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go new file mode 100644 index 00000000..c68914e4 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself +// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools. +func TestEveryStateAndActHasItsTool(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + State []string `json:"state"` + Logs []string `json:"logs"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + via := map[string]string{ + ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list", + ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history", + DeclaredState: DeclaredTool, TakeoverState: DeclaredTool, + PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool, + ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool, + IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool, + "a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool, + } + served := map[string]bool{} + for _, tool := range Tools(&Config{}) { + served[tool.Name] = true + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") { + t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs) + } + for _, verb := range SeatVerbs { + if !served[Seat+"."+verb] { + t.Errorf("the seat's verb %s is not served", verb) + } + } + for _, name := range m.Tools { + if !served[name] { + t.Errorf("the manifest lists the tool %s, which is not served", name) + } + } + for _, list := range [][]string{m.State, m.Logs, m.Reads} { + for _, s := range list { + if _, ok := via[s]; !ok { + t.Errorf("%s has no tool that reads it", s) + } + } + } + for what, tool := range via { + name, _, _ := strings.Cut(tool, " ") + if name == "config" { + name = Seat + "." + Verb + } + if !served[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index d6cb304d..18619024 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) { for _, r := range ready { said = append(said, c.decide(r.p, r.w)) } + said = append(said, c.decideTakeovers()...) c.recordWarrants(now) return said, nil } diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go index f803b5de..8c0647a6 100644 --- a/modules/mesh-config/cmd/mesh-config/pulls.go +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -10,6 +10,8 @@ import ( "strconv" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826). @@ -61,31 +63,22 @@ type PullDecl struct { At time.Time `json:"at"` } -// KeptValue is an approved held value, kept apart from the values mesh-config resolves. -type KeptValue struct { - Pull string `json:"pull"` - Head string `json:"head"` - Digest string `json:"declaration"` - Node string `json:"node"` - Module string `json:"module"` - Key string `json:"key"` - Value json.RawMessage `json:"value"` - By string `json:"by"` - Via string `json:"via"` - Why string `json:"why"` - // Approved is who approved it, as the proposal kept it. - Approved string `json:"approved"` - Proposal string `json:"proposal"` - At time.Time `json:"at"` -} +// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the +// checker and the assigner's composition check read (novox/hq ADR 0565 §5). +type KeptValue = delivery.KeptValue +// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is +// not one. func keptKey(pull, node, module, key string) string { + if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil { + return k + } return pullKey(pull) + "." + node + "." + module + "." + key } -func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } +func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } -func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } +func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } // declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same. func declDigest(d Declaration) string { @@ -151,6 +144,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+ "mesh-config what the head declares; %s is not the controller", call.Caller) } + if sw, err := c.source(); err != nil { + return nil, err + } else if sw.Source == SourceIndexer { + return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " + + "declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)") + } if _, _, _, err := parsePull(call.Pull); err != nil { return nil, err } @@ -185,6 +184,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { pull := strings.TrimSpace(call.Pull) c.mu.Lock() defer c.mu.Unlock() + return c.declarePull(pull, head, declares) +} + +// declarePull keeps what a pull request's head declares and judges again the values held for it; from the +// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock. +func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) { if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares, At: c.now()}); err != nil { return nil, fmt.Errorf("the declarations were not kept: %v", err) @@ -235,7 +240,7 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { // heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and // config-changed; what could not be done is answered, not dropped. func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string { - level := kv.level() + level := keptLevel(kv) ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why, Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}} unsaid := c.record(level.ValueKey(kv.Key), ch) @@ -252,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string // dropKept removes a held value, saying why. func (c *Config) dropKept(kv KeptValue, why string) []string { - if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil { - return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()} + if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil { + return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()} } c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why) return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "") @@ -408,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) { kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module, Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID, At: c.now()} - if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil { + if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil { return "", fmt.Errorf("the value was not kept: %v", err) } unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved) @@ -471,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string { said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...) continue } - vk := kv.level().ValueKey(kv.Key) + vk := keptLevel(kv).ValueKey(kv.Key) v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull + ", merged)", Approved: kv.Approved, At: c.now()} if err := c.Store.Put(ValuesState, vk, v); err != nil { diff --git a/modules/mesh-config/cmd/mesh-config/reads-outside b/modules/mesh-config/cmd/mesh-config/reads-outside new file mode 100644 index 00000000..ca488b77 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-config/module.json diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index 761706d8..e32b4dda 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -12,10 +12,19 @@ import ( // Reading the sources, writing the resolved values, and publishing the list of predefined keys. -// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the -// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the -// controller (issue 1953, ADR 0565 §3). +// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names +// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations +// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror. func (c *Config) sources() (Sources, error) { + sw, err := c.source() + if err != nil { + return Sources{}, err + } + return c.sourcesFrom(sw.Source) +} + +// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror. +func (c *Config) sourcesFrom(source string) (Sources, error) { s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{}, Predefined: map[string]Published{}, Values: map[string]Value{}} var problems []string @@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) { if err := c.readDeclarations(&s); err != nil { return s, fmt.Errorf("the modules' declarations cannot be read: %w", err) } + if source == SourceController { + c.mirrorDeclarations(&s) + } for _, u := range s.Unresolved { problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why)) } @@ -350,6 +362,10 @@ type Resolution struct { Moved []Moved `json:"moved"` Published string `json:"published,omitempty"` Withheld string `json:"withheld,omitempty"` + // Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates + // did after the take-over. + Declared []string `json:"declared,omitempty"` + Candidates []string `json:"candidates,omitempty"` } // Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources @@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) { c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " + "keys changed", At: c.now(), Moved: moved}) } + if written, derr := c.writeDeclared(s); derr != nil { + c.logf("[mesh-config] declared is not all written: %v", derr) + } else if len(written) > 0 { + r.Declared = written + } + if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer { + r.Candidates = c.takeCandidates() + } return r, err } -// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published -// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the +// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the +// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins; +// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the // list published stays, and why is said in every listing and in the log, because a value for a key no longer on // the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused). // It answers what it published, empty when nothing changed. func (c *Config) publish() (string, error) { - e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + sw, err := c.source() if err != nil { - return "", fmt.Errorf("the controller's list cannot be read: %w", err) + return "", err } - if e == nil { + read, whose := c.mirrorList, "the controller's list" + if sw.Source == SourceIndexer { + read, whose = c.indexerList, "the indexer's list" + } + list, found, err := read() + if err != nil { + return "", fmt.Errorf("%s cannot be read: %w", whose, err) + } + if !found { return "", nil // no list kept yet } - var list PredefinedList - if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { - return "", fmt.Errorf("the controller's list is not one: %s", e.Value) - } current := map[string]Published{} if err := c.each(PredefinedState, func(key string, raw []byte) error { var p Published @@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) { }); err != nil { return "", err } - generation := 0 + generation, sameCommit := 0, len(current) > 0 for _, p := range current { generation = max(generation, p.Generation) + sameCommit = sameCommit && p.Commit == list.Commit } - if len(current) > 0 && generation >= list.Generation { + if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) { c.withheld = "" return "", nil } diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go new file mode 100644 index 00000000..7fbc2ffc --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -0,0 +1,79 @@ +package main + +// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs +// and as the module's own tools beside them, the read of `declared` and resync. + +import ( + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The switch's tools. +const ( + CompareTool = "config_compare" + TakeOverTool = "config_take_over" + HandBackTool = "config_hand_back" + DeclaredTool = "config_declared" + ResyncTool = "config_resync" +) + +// SeatVerbs are the seat's verbs, as the manifest claims them. +var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb} + +func switchTools(c *Config) []stdio.Tool { + own := ownSwitchTools(c) + byName := map[string]stdio.Tool{} + for _, t := range own { + byName[t.Name] = t + } + for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} { + t := byName[tool] + t.Name = Seat + "." + verb + own = append(own, t) + } + return own +} + +func ownSwitchTools(c *Config) []stdio.Tool { + why := map[string]any{"type": "object", "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}} + whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: CompareTool, + Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " + + "each placed module's declarations (declared against the mirror declared-config), the list of predefined " + + "keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " + + "against the controller's pull-declares), and every placed module's value on every node resolved both " + + "ways; each with its reason. The take-over is refused while any stands.", + Run: func(map[string]any) (any, error) { return c.Compare() }}, + {Name: TakeOverTool, + Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " + + "the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " + + "Refused while compare shows a difference; on Approve switched if nothing differs then.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: HandBackTool, + Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " + + "pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: DeclaredTool, + Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " + + "commit they come from and how many keys, or one module whole; and which source holds now.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module, whole"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + return c.Declared(strings.TrimSpace(m)) + }}, + {Name: ResyncTool, + Description: "Read every source again and write what differs: the published list of predefined keys, the " + + "resolved values, declared, and after the take-over the indexer's config candidates.", + Run: func(map[string]any) (any, error) { return c.Resync() }}, + } +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover.go b/modules/mesh-config/cmd/mesh-config/takeover.go new file mode 100644 index 00000000..dad88461 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover.go @@ -0,0 +1,771 @@ +package main + +// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of +// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every +// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list, +// each open pull request's config declarations, and every placed module's resolved value on every node both ways; +// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the +// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write. +// +// **Which source holds** is the key `switch` alone, read by mesh-config alone: +// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list +// of predefined keys from the controller's mirror, a pull request's declarations from the controller's +// pull-declares; +// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from +// the indexer's `config-candidates`; pull-declares refused; +// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror, +// pull-declares taken. + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The states and reads of the switch. +const ( + // DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3). + DeclaredState = "declared" + // TakeoverState holds the key SwitchKey and each take-over ask, `ask.`. + TakeoverState = "takeover" + SwitchKey = "switch" + + // MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module: + // what `compare` holds the own source to, and the source after a hand-back. + MirrorDeclaredRead = "mesh-controller.declared-config" + // IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2). + IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState + CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState + // LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13). + LeaseRead = buscompose.ControllerLease +) + +// The sources the switch names. +const ( + SourceInterim = "interim" + SourceIndexer = "indexer" + SourceController = "controller" +) + +// TakeOverVerb and HandBackVerb are the seat's verbs. +const ( + TakeOverVerb = "take-over" + HandBackVerb = "hand-back" + CompareVerb = "compare" +) + +// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask. +type Switch struct { + Source string `json:"source"` + At time.Time `json:"at"` + By string `json:"by"` + Why string `json:"why"` + Ask string `json:"ask,omitempty"` +} + +// source is the source that holds now. +func (c *Config) source() (Switch, error) { + e, err := c.Store.Get(TakeoverState, SwitchKey) + if err != nil { + return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err) + } + if e == nil { + return Switch{Source: SourceInterim}, nil + } + var s Switch + if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) { + return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value) + } + return s, nil +} + +// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading). +func (c *Config) leaseAway() bool { + e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +} + +// ---- declared -------------------------------------------------------------------------------------------- + +// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.` +// names holds them, with that manifest's digest and commit (ADR 0565 §3). +type DeclaredKept struct { + Module string `json:"module"` + Manifest string `json:"manifest"` + Commit string `json:"commit,omitempty"` + Config map[string]Declaration `json:"settings"` +} + +// trunkKept is a module's declarations at the trunk with the manifest they come from. +func (c *Config) trunkKept(module string) (DeclaredKept, error) { + key, err := delivery.ModulesKey(module) + if err != nil { + return DeclaredKept{}, err + } + e, err := c.Store.Get(ModulesRead, key) + if err != nil || e == nil { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err) + } + var m struct { + Manifest string `json:"manifest"` + Commit string `json:"commit"` + } + if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module) + } + d, err := c.manifestDeclarations(module, m.Manifest) + if err != nil { + return DeclaredKept{}, err + } + return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil +} + +// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs +// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written. +func (c *Config) writeDeclared(s Sources) ([]string, error) { + held := map[string][]byte{} + if err := c.each(DeclaredState, func(key string, raw []byte) error { + held[key] = raw + return nil + }); err != nil { + return nil, err + } + var written []string + var problems []error + placed := map[string]bool{} + for _, module := range modulesOf(s.Assignments) { + placed[module] = true + k, err := c.trunkKept(module) + if err != nil { + continue // unresolved, said by the resolution; its key stays as it was + } + raw, _ := json.Marshal(k) + if bytes.Equal(raw, held[module]) { + continue + } + if err := c.Store.Put(DeclaredState, module, k); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + continue + } + written = append(written, module) + } + for module := range held { + if !placed[module] { + if err := c.Store.Delete(DeclaredState, module); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + } + } + } + return written, errors.Join(problems...) +} + +// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source +// after a hand-back, and the other side of compare. +func (c *Config) mirrorDeclarations(s *Sources) { + s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil + for _, module := range modulesOf(s.Assignments) { + e, err := c.Store.Get(MirrorDeclaredRead, module) + var d ModuleDeclarations + if err == nil && e != nil { + d, err = ParseModuleDeclarations(e.Value) + } else if err == nil { + err = errors.New("the mirror holds no key for it") + } + if err != nil { + for _, a := range s.Assignments { + if a.Module == module { + s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module, + Why: "the controller's mirror declared-config: " + err.Error()}) + } + } + continue + } + s.Declared[module] = d + } +} + +// ---- the predefined list and the pull requests from the indexer ------------------------------------------ + +// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit. +func (c *Config) indexerList() (PredefinedList, bool, error) { + e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var kept delivery.PredefinedKept + if err := json.Unmarshal(e.Value, &kept); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err) + } + list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit} + if err := json.Unmarshal(kept.List, &list.Keys); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err) + } + return list, true, nil +} + +// mirrorList is the controller's mirror's list. +func (c *Config) mirrorList() (PredefinedList, bool, error) { + e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var list PredefinedList + if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { + return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value) + } + return list, true, nil +} + +// candidates is every open pull request's config declarations as the indexer keeps them, by pull key. +func (c *Config) candidates() (map[string]PullDecl, error) { + out := map[string]PullDecl{} + err := c.each(CandidatesRead, func(key string, raw []byte) error { + var cand delivery.ConfigCandidate + if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" { + return fmt.Errorf("the indexer's config candidate %s is not one", key) + } + p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At} + for module, keys := range cand.Declares { + raw, _ := json.Marshal(keys) + var d map[string]Declaration + if err := json.Unmarshal(raw, &d); err != nil { + return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module) + } + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } + d[k] = decl + } + p.Declares[module] = d + } + out[pullKey(cand.Pull)] = p + return nil + }) + return out, err +} + +// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one +// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502 +// §6). The caller holds the lock. +func (c *Config) takeCandidates() []string { + cands, err := c.candidates() + if err != nil { + return []string{"the indexer's config candidates cannot be read: " + err.Error()} + } + var s Sources + if err := c.readPulls(&s); err != nil { + return []string{err.Error()} + } + var said []string + for _, k := range sortedKeys(cands) { + p := cands[k] + if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) { + continue + } + a, err := c.declarePull(p.Pull, p.Head, p.Declares) + if err != nil { + said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err)) + continue + } + said = append(said, a.Said+" (from the indexer)") + } + return said +} + +// declaresDigest is what a pull request's declarations hold a value to, in one encoding. +func declaresDigest(d map[string]map[string]Declaration) string { + flat := map[string]string{} + for m, keys := range d { + for k, decl := range keys { + flat[m+"."+k] = declDigest(decl) + } + } + raw, _ := json.Marshal(flat) + return string(raw) +} + +// ---- compare ---------------------------------------------------------------------------------------------- + +// Difference is one thing the own sources and the controller's mirrors say differently, and why. +type Difference struct { + What string `json:"what"` + Module string `json:"module,omitempty"` + Node string `json:"node,omitempty"` + Key string `json:"key,omitempty"` + Pull string `json:"pull,omitempty"` + Own json.RawMessage `json:"own,omitempty"` + Mirror json.RawMessage `json:"mirror,omitempty"` + Why string `json:"why"` +} + +// The kinds of difference. +const ( + DiffDeclared = "declared" + DiffPredefined = "predefined" + DiffPull = "pull" + DiffValue = "value" +) + +// Comparison is what compare answers. +type Comparison struct { + Source string `json:"source"` + Differences []Difference `json:"differences"` + Count int `json:"count"` + Said string `json:"said"` +} + +// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock. +func (c *Config) Compare() (*Comparison, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.compare() +} + +// compare is Compare; the caller holds the lock. +func (c *Config) compare() (*Comparison, error) { + sw, err := c.source() + if err != nil { + return nil, err + } + own, err := c.sourcesFrom(SourceInterim) + if err != nil { + return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err) + } + mirror := own + mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil + c.mirrorDeclarations(&mirror) + var diffs []Difference + + // Each placed module's declarations: `declared` against the mirror's key for it. + for _, module := range modulesOf(own.Assignments) { + var ownRaw []byte + if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil { + var k DeclaredKept + if json.Unmarshal(e.Value, &k) == nil { + ownRaw, _ = json.Marshal(normal(k.Config)) + } + } + var mirrorRaw []byte + if d, ok := mirror.Declared[module]; ok { + mirrorRaw, _ = json.Marshal(normal(d.Config)) + } + switch { + case ownRaw == nil && mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"}) + case ownRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw, + Why: "the mirror holds its declarations and declared does not"}) + case mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, + Why: "declared holds its declarations and the mirror does not"}) + case !bytes.Equal(ownRaw, mirrorRaw): + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw, + Why: "the trunk's manifest declares it otherwise than the mirror"}) + } + } + + // The predefined list: the indexer's against the mirror's. + il, iok, ierr := c.indexerList() + ml, mok, merr := c.mirrorList() + switch { + case ierr != nil || merr != nil: + diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()}) + case !iok || !mok: + if iok != mok { + diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+ + "mirror holds one: %v", iok, mok)}) + } + default: + for _, k := range unionKeys(il.Keys, ml.Keys) { + a, aok := il.Keys[k] + b, bok := ml.Keys[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"}) + case !bytes.Equal(ar, br): + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br, + Why: "the two lists declare it otherwise"}) + } + } + } + + // Each open pull request: the indexer's candidate against what the controller told by pull-declares. + cands, cerr := c.candidates() + if cerr != nil { + diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()}) + } + for _, k := range unionKeys(cands, own.Pulls) { + a, aok := cands[k] + b, bok := own.Pulls[k] + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"}) + case a.Head != b.Head: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+ + "the controller told %s", short(a.Head), short(b.Head))}) + case declaresDigest(a.Declares) != declaresDigest(b.Declares): + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"}) + } + } + + // Every placed module's value on every node, resolved both ways. + ownValues, mirrorValues := Resolve(own), Resolve(mirror) + for _, k := range unionKeys(ownValues, mirrorValues) { + a, aok := ownValues[k] + b, bok := mirrorValues[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + node, module, key := a.Node, a.Module, a.Key + if !aok { + node, module, key = b.Node, b.Module, b.Key + } + d := Difference{What: DiffValue, Node: node, Module: module, Key: key} + switch { + case !aok: + d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node" + case !bok: + d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it" + case !bytes.Equal(ar, br): + d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source" + default: + continue + } + diffs = append(diffs, d) + } + for _, u := range mirror.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why}) + } + for _, u := range own.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why}) + } + sort.SliceStable(diffs, func(i, j int) bool { + a, b := diffs[i], diffs[j] + return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull + }) + if diffs == nil { + diffs = []Difference{} + } + cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)} + if len(diffs) == 0 { + cmp.Said = "nothing differs: the take-over may be asked" + } else { + cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs)) + } + return cmp, nil +} + +// normal is declarations in one encoding: a null default as none. +func normal(d map[string]Declaration) map[string]Declaration { + out := map[string]Declaration{} + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } else { + decl.Default = compact(decl.Default) + } + out[k] = decl + } + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// ---- take-over and hand-back --------------------------------------------------------------------------------- + +// TakeoverAsk is one take-over asked of the operator, `takeover.ask.`: what the ask is composed from again, and +// where it stands. +type TakeoverAsk struct { + ID string `json:"id"` + From string `json:"from"` + Why string `json:"why"` + At time.Time `json:"at"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Decided time.Time `json:"decided,omitzero"` + Answered string `json:"answered,omitempty"` + Recorded bool `json:"recorded,omitempty"` +} + +func askKey(id string) string { return "ask." + id } + +// act is what an answer to the take-over binds. +func (t TakeoverAsk) act(option string) asks.Act { + return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From, + "at": t.At.UTC().Format(time.RFC3339Nano)} +} + +// ask is the take-over's ask, composed from it alone. +func (t TakeoverAsk) ask() asks.Ask { + why, _ := sayable("Why: " + t.Why) + a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources", + Explanation: strings.Join([]string{ + "mesh-config would read the list of predefined keys and each pull request's config from the delivery's " + + "indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.", + point(why)}, "\n"), + Who: asks.Operator, Expires: t.At.Add(ProposalLasts), + OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover", + Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " + + "to the controller's records until the controller's lease is handed away."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"}, + {OptionDecline, "Decline", "nothing changes"}, + } { + binds, _ := asks.ActDigest(t.act(o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// TakeoverAnswer is what take-over and hand-back answer. +type TakeoverAnswer struct { + Said string `json:"said"` + Ask string `json:"ask,omitempty"` + Source string `json:"source"` +} + +// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference. +func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why the take-over is asked now: it is shown to the operator") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + if c.Asks == nil { + return nil, errors.New("the operator cannot be asked from here, so nothing is switched") + } + c.mu.Lock() + defer c.mu.Unlock() + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceIndexer { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " + + sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil + } + cmp, err := c.compare() + if err != nil { + return nil, err + } + if cmp.Count > 0 { + first := cmp.Differences[0] + return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+ + "Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why) + } + now := c.now() + var open []string + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting && + now.Before(t.At.Add(ProposalLasts)) { + open = append(open, t.ID) + } + return nil + }) + if len(open) > 0 { + return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0]) + } + t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(), + State: ProposalWaiting} + ask := t.ask() + if err := ask.Check(now); err != nil { + return nil, err + } + if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil { + return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err) + } + if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil { + t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error() + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return nil, fmt.Errorf("the operator could not be asked: %w", err) + } + return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+ + "mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+ + "then", ask.Headline)}, nil +} + +// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock. +func (c *Config) decideTakeovers() []string { + var said []string + var waiting []TakeoverAsk + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting { + waiting = append(waiting, t) + } + return nil + }) + now := c.now() + for _, t := range waiting { + keep := func(state, outcome string) string { + t.State, t.Outcome, t.Decided = state, outcome, now + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return "take-over " + t.ID + ": " + outcome + } + e, err := c.Asks.Record(t.ID) + if err != nil || e == nil { + if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) { + said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched")) + } + continue + } + var rec routerRecord + if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != t.ID { + said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched")) + continue + } + if w.Outcome != asks.OutcomeChosen { + said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched")) + continue + } + option, err := w.For(Module, t.ask()) + if err == nil { + err = option.Performs(t.act(option.ID)) + } + if err != nil { + said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error())) + continue + } + t.Answered = w.Says() + if option.ID != OptionApprove { + said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched")) + continue + } + cmp, err := c.compare() + switch { + case err != nil: + said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error())) + continue + case cmp.Count > 0: + said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+ + "now; nothing was switched, ask again once it is empty", cmp.Count))) + continue + } + sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID} + if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil { + said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+ + "on the next look", t.ID, err)) + continue + } + if err := c.Asks.Warranted(t.ID); err == nil { + t.Recorded = true + } + said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now")) + } + return said +} + +func approvedTakeover(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) +} + +// HandBack returns to the controller's records until the controller's lease is handed away. +func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why config hands back: it is kept with the switch") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + c.mu.Lock() + defer c.mu.Unlock() + if c.leaseAway() { + return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " + + "(novox/hq ADR 0565 §4)") + } + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceController { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil + } + now := c.now() + back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)} + if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil { + return nil, fmt.Errorf("the switch could not be written: %w", err) + } + answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " + + "declared-config and predefined, and pull-declares"} + if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil { + answer.Said += "; the hand-act log does not hold it: " + err.Error() + } + return answer, nil +} + +// Declared is the read of `declared`: every module's key, or one whole. +func (c *Config) Declared(module string) (any, error) { + if module != "" { + e, err := c.Store.Get(DeclaredState, module) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module) + } + return json.RawMessage(e.Value), nil + } + rows := map[string]any{} + err := c.each(DeclaredState, func(key string, raw []byte) error { + var k DeclaredKept + if json.Unmarshal(raw, &k) == nil { + rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)} + } + return nil + }) + sw, _ := c.source() + return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err +} + +// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and +// after the take-over the indexer's config candidates (ADR 0565 §15). +func (c *Config) Resync() (any, error) { + r, err := c.Reconcile() + if err != nil { + return r, err + } + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]any{"resolution": r} + if sw, err := c.source(); err == nil && sw.Source == SourceIndexer { + out["candidates"] = c.takeCandidates() + } + return out, nil +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover_test.go b/modules/mesh-config/cmd/mesh-config/takeover_test.go new file mode 100644 index 00000000..c66b21f8 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations. +const ( + claudeDeclared = `{"module":"claude-code","settings":{ + "managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}` + oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}` +) + +// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's +// declarations of every placed module, and the indexer's predefined list equal to the mirror's. +func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) { + t.Helper() + c, m, r := aProposingMesh(t) + m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared) + m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared) + m.mirror(t, MirrorDeclaredRead, "old", oldDeclared) + indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + return c, m, r +} + +// indexerList is the indexer's `predefined.list` at a commit. +func indexerList(t *testing.T, m *memStore, commit, list string) { + t.Helper() + raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)}) + m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw)) +} + +// candidate is the indexer's config candidate of a pull request. +func candidate(t *testing.T, m *memStore, pull, head, declares string) { + t.Helper() + key, err := delivery.ConfigCandidateKey(pull) + if err != nil { + t.Fatal(err) + } + m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`) +} + +func kinds(cmp *Comparison) []string { + var out []string + for _, d := range cmp.Differences { + k := d.What + ":" + d.Module + d.Pull + d.Key + if d.Node != "" { + k += "@" + d.Node + } + if !slices.Contains(out, k) { + out = append(out, k) + } + } + return out +} + +// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's +// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and +// its insight of 2026-10-29; issue 1953 L5-C4). +func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) { + cases := []struct { + name string + change func(t *testing.T, m *memStore) + want []string + }{ + {"all agree", func(*testing.T, *memStore) {}, nil}, + {"the mirror declares a default otherwise", func(t *testing.T, m *memStore) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + }, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}}, + {"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") }, + []string{"declared:old", "value:old@g14", "value:oldworkers@g14"}}, + {"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) { + m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1)) + }, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558) + {"the lists differ", func(t *testing.T, m *memStore) { + indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + }, []string{"predefined:agent-managed-settings"}}, + {"a pull request only the indexer holds", func(t *testing.T, m *memStore) { + candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + }, []string{"pull:novox/mesh-catalog#9"}}, + } + for _, tc := range cases { + c, m, r := anAgreeingMesh(t) + tc.change(t, m) + cmp, err := c.Compare() + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if got := kinds(cmp); !slices.Equal(got, tc.want) { + t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences) + } + for _, d := range cmp.Differences { + if d.Why == "" { + t.Errorf("%s: a difference without its reason: %+v", tc.name, d) + } + } + a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"}) + if len(tc.want) > 0 { + if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 { + t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err) + } + continue + } + if err != nil || a.Ask == "" || len(r.subjects) != 1 { + t.Errorf("%s: the take-over was not asked: %v", tc.name, err) + } + } +} + +// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests' +// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the +// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13). +func TestTheTakeOverAndTheHandBack(t *testing.T) { + c, m, r := anAgreeingMesh(t) + steps := []struct { + name string + do func() (string, error) + want string // what the answer or the error says + }{ + {"declared is written when it differs, and once", func() (string, error) { + _ = m.Delete(DeclaredState, "dunst") + first, err := c.Reconcile() + again, _ := c.Reconcile() + return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err + }, "dunst|"}, + {"pull-declares taken before the take-over", func() (string, error) { + a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0", + Declares: map[string]any{}}) + return answered(a), err + }, "keeps what"}, + {"the take-over is asked", func() (string, error) { + _ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate + a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"}) + return answered(a), err + }, "is asked"}, + {"the operator approves: switched to the indexer", func() (string, error) { + r.choose(t, r.lastAsk(t), OptionApprove, nil) + out, err := c.Decide() + return strings.Join(out, "; "), err + }, "reads its own sources from now"}, + {"pull-declares refused after it, naming the indexer", func() (string, error) { + _, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1", + Declares: map[string]any{}}) + return "", err + }, "config-candidates"}, + {"the list comes from the indexer", func() (string, error) { + indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72}, + "agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "theme":{"type":"string","description":"the desktop's theme"}}`) + res, err := c.Reconcile() + return res.Published, err + }, "theme"}, + {"a pull request's config comes from the indexer", func() (string, error) { + candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + res, err := c.Reconcile() + return strings.Join(res.Candidates, "; "), err + }, "from the indexer"}, + {"hand-back returns to the mirrors", func() (string, error) { + a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"}) + return answered(a), err + }, "controller's records again"}, + {"after it, declarations resolve from the mirror", func() (string, error) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + _, err := c.Reconcile() + return string(m.resolved(t, "g14", "dunst", "width").Value), err + }, "400"}, + {"hand-back is gone after the lease write", func() (string, error) { + m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`) + _, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"}) + return "", err + }, "gone"}, + {"the terminal sets no trusted value after the lease write", func() (string, error) { + _, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{}, + Why: "x", Caller: ControllerCaller, By: "person.jochen"}) + return "", err + }, "propose"}, + } + for _, s := range steps { + got, err := s.do() + text := got + if err != nil { + text = err.Error() + } + if !strings.Contains(text, s.want) { + t.Errorf("%s: said %q, want %q", s.name, text, s.want) + } + } + if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") { + t.Errorf("the hand-back is not recorded: %v", m.handActs) + } + if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) { + t.Errorf("declared holds no dunst with its manifest: %v", e) + } + // An Approve that finds compare differing at the answer switches nothing. + c2, m2, r2 := anAgreeingMesh(t) + if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil { + t.Fatal(err) + } + m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + r2.choose(t, r2.lastAsk(t), OptionApprove, nil) + out, _ := c2.Decide() + if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") { + t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out) + } +} + +// lastAsk is the newest ask the router took. +func (r *standIn) lastAsk(t *testing.T) string { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + var newest string + var at time.Time + for id, a := range r.asks { + if newest == "" || a.Expires.After(at) { + newest, at = id, a.Expires + } + } + if newest == "" { + t.Fatal("the router took no ask") + } + return newest +} + +func answered(a any) string { + raw, _ := json.Marshal(a) + return string(raw) +} diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 6a0791d7..0c8a0a52 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,4 +2,13 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 8c9ba7ca..05084f7c 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,2 +1,26 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 66db5b60..9554db29 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -7,7 +7,10 @@ "name": "mesh-config", "scope": "mesh", "serves": [ - "config" + "config", + "compare", + "take-over", + "hand-back" ] } ], @@ -23,6 +26,8 @@ "gitea.forge.pull-request.closed" ], "invokes": [ + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", "seat:mesh-controller.hand-act", "seat:mesh-controller.warranted", "gitea.gitea_get_pull_request" @@ -33,7 +38,18 @@ "mesh-delivery-builder.builds", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.manifests", - "mesh-controller.predefined" + "mesh-controller.predefined", + "mesh-controller.declared-config", + "mesh-delivery-indexer.predefined", + "mesh-delivery-indexer.config-candidates", + "mesh-controller.lease" + ], + "tools": [ + "config_compare", + "config_take_over", + "config_hand_back", + "config_declared", + "config_resync" ], "state": [ "values", @@ -41,7 +57,9 @@ "predefined", "proposals", "pulls", - "kept" + "kept", + "declared", + "takeover" ], "logs": [ "history" diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 2a7ba773..bfbcfae4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -330,7 +330,7 @@ func (h *Holder) endStaleAsks() { h.endStaleSettles() for _, d := range h.deliveries { r := d.Ready - if r == nil || r.Merged != nil || d.MergedAs != "" || r.Declined != nil || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { + if r == nil || r.Merged != nil || d.MergedAs != "" || r.declined() || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { continue } switch { @@ -418,6 +418,7 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { h.runOf(m, now) m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) m.Ready.AskedKey, m.Ready.Asking, m.Ready.AskCheck = id, asking, check.Key + m.Ready.Declined, m.Ready.AnsweredAsk = nil, "" // a new ask: an answer to an earlier one does not stand on it h.keep(m) taken = append(taken, m) reads = append(reads, readOf(m, m.Commit)) @@ -608,7 +609,7 @@ func (h *Holder) pendingAsks() []string { seen := map[string]bool{} var out []string for _, d := range h.deliveries { - if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil && !seen[r.AskID] { + if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && !r.declined() && !seen[r.AskID] { seen[r.AskID] = true out = append(out, r.AskID) } @@ -705,10 +706,10 @@ func (h *Holder) answered(key string, ans sdk.Answer) []func() { h.mergeAgainNow(d) // the operator chose: the merge is asked at once switch ans.Option { case OptionApprove: - r.Approved = &now + r.Approved, r.Declined, r.AnsweredAsk = &now, nil, key h.Logf("[mesh-delivery] %s: the operator approved its merge into main (%s)", d.ID, r.Warrant) case OptionDecline: - r.Declined = &now + r.Declined, r.AnsweredAsk = &now, key h.Logf("[mesh-delivery] %s: the operator declined its merge into main (%s)", d.ID, r.Warrant) said = append(said, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** The merge of this pull request into main at %s was "+ "declined: %s. A new head asks the operator again.", d.Commit, r.Warrant))) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go index fde6b6e7..3057f3d8 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go @@ -405,3 +405,51 @@ func TestAnAnswerToAnEarlierAskingIsNotTakenAndTheMergeNamesTheAnsweredAsking(t t.Fatalf("the merge is announced %v; wanted the ask id of asking 2, %s", say, sdk.AskingID(key, 2)) } } + +// 2026-10-29 (hq#495): a group declined under one ask, a member pushed, and the group asked and approved again at the new +// heads: the member whose head did not move kept the earlier Decline beside the new Approve, and nothing was handed, with +// no word. An answer stands only for the ask it answers; whichever member is pushed, both merge on the new Approve. +func TestADeclineOfAnEarlierAskDoesNotHoldTheApproveOfTheNext(t *testing.T) { + const ctlHead, catHead, pushed = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2", "e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3" + for _, push := range []struct { + repo string + number int + trunk string + }{{"novox/mesh-catalog", 2, "t3t3t3t3t3t3"}, {"novox/mesh-controller", 1, "t1t1t1t1t1t1"}} { + t.Run(push.repo, func(t *testing.T) { + w := newWorld(t) + w.op.auto = false + heads := map[string]string{"novox/mesh-controller": ctlHead, "novox/mesh-catalog": catHead} + paths := map[string]string{"novox/mesh-controller": "internal/a.go", "novox/mesh-catalog": "modules/app/module.json"} + numbers := map[string]int{"novox/mesh-controller": 1, "novox/mesh-catalog": 2} + trunks := map[string]string{"novox/mesh-controller": "t1t1t1t1t1t1", "novox/mesh-catalog": "t3t3t3t3t3t3"} + round := func(check string) { + var open []OpenPull + for _, repo := range []string{"novox/mesh-controller", "novox/mesh-catalog"} { + w.h.PullUpdated(pr(repo, numbers[repo], heads[repo], "feat/x", paths[repo])) + w.h.Checked(judgedAt(repo, numbers[repo], heads[repo], "pass", trunks[repo])) + open = append(open, openPull(repo, numbers[repo], heads[repo])) + } + w.forge.open = open + w.h.SeeOpenPulls() + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/x", ID: check, Verdict: "pass", Summary: "both compose"}) + w.settleAll() + } + round("check-1") + w.op.answer(w.op.open()[0], OptionDecline) + w.settleAll() + heads[push.repo] = pushed + round("check-2") + open := w.op.open() + if len(open) != 1 { + t.Fatalf("the group at its new heads is not asked once: %v", open) + } + w.op.answer(open[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 2 { + t.Fatalf("approved at the new heads, yet merged %v; stalled %+v", w.forge.merges, w.h.Stalled()) + } + }) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go index b0fc97af..be95a0c7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go @@ -96,7 +96,7 @@ func (h *Holder) withdrawForStatus(e ForgeStatusEvent) { done := map[string]bool{} for _, d := range h.deliveries { r := d.Ready - if !strings.EqualFold(d.Repository, repository) || r == nil || r.AskID == "" || r.Approved != nil || r.Declined != nil || done[r.AskID] { + if !strings.EqualFold(d.Repository, repository) || r == nil || r.AskID == "" || r.Approved != nil || r.declined() || done[r.AskID] { continue } head := d.Commit diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go index 8b2fd6a2..b6e1897c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go @@ -208,7 +208,7 @@ func (h *Holder) passes() []pass { for _, id := range ids { d := h.deliveries[id] r := d.Ready - if r == nil || r.Approved == nil || r.Declined != nil || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || + if r == nil || r.Approved == nil || r.declined() || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || d.Base != MergeBranch || h.allMerged(r.Heads) { continue } @@ -335,7 +335,7 @@ func (h *Holder) MergeAgain(id, why string) (string, error) { return "", fmt.Errorf("say why: the merge asked again is the operator's word, kept in the log") } c := h.covering(d) - if c == nil || c.Approved == nil || c.Declined != nil || c.Heads[d.ID] != d.Commit { + if c == nil || c.Approved == nil || c.declined() || c.Heads[d.ID] != d.Commit { return "", fmt.Errorf("%s is not approved at its head %s: nothing to ask again (a new head is asked on its own)", id, shortOf(d.Commit, 12)) } if d.MergedAs != "" || (d.Ready != nil && d.Ready.Merged != nil) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 79af7cda..8ba98d5c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -524,7 +524,7 @@ func (h *Holder) PullEdited(p PullEvent) { if after, _ := reviewWords(d); after == before { return } - if r := h.covering(d); r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil { + if r := h.covering(d); r != nil && r.AskID != "" && r.Approved == nil && !r.declined() { h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, d.ID+"'s review record changed: "+shortOf(d.ReviewedAt, 12))) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index ec1b6cec..d2634300 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -149,11 +149,10 @@ func (h *Holder) handTurn(g *Group, d *Delivery, now time.Time) func() { return nil } for _, m := range order { - mc := h.covering(m) - if m.Ready == nil || m.Ready.Declined != nil || mc == nil || mc.Approved == nil || mc.Heads[m.ID] != m.Commit || h.readyWaits(m) != "" { - return nil // the others are not approved at their heads yet: nothing is merged before they are + if h.notHandable(m) != "" { + return nil // the others are not approved at their heads yet: nothing is merged before they are (said by notHanded) } - reads = append(reads, readOf(m, mc.Heads[m.ID])) + reads = append(reads, readOf(m, h.covering(m).Heads[m.ID])) } } if c == nil || c.Approved == nil || c.Heads[d.ID] == "" { @@ -169,6 +168,66 @@ func (h *Holder) handTurn(g *Group, d *Delivery, now time.Time) func() { return func() { h.hand(gid, name, key, target, reads, approved) } } +// notHandable is why a group member cannot be handed in its group's pass, empty when it can: handTurn's own test, in words. +// Under the lock. +func (h *Holder) notHandable(m *Delivery) string { + mc := h.covering(m) + switch { + case m.Ready == nil: + return m.ID + " has no ready run (" + string(m.State) + ")" + case m.Ready.declined(): + return m.ID + " stands declined since " + m.Ready.Declined.UTC().Format(time.RFC3339) + case mc == nil || mc.Approved == nil: + return m.ID + " is not approved at its head " + shortOf(m.Commit, 12) + case mc.Heads[m.ID] != m.Commit: + return m.ID + "'s head " + shortOf(m.Commit, 12) + " is not the head approved" + } + if w := h.readyWaits(m); w != "" { + return m.ID + " waits: " + w + } + return "" +} + +// notHanded is why the approved merge of d, which waits for nothing, is not handed: its group's pass cannot start, and +// says which member holds it and why. Empty when nothing holds it, or when the pass is under way or its refusal is said +// already. Under the lock. +func (h *Holder) notHanded(g *Group, d *Delivery) string { + if g == nil { + return "" + } + order := h.unmergedLive(g) + if len(order) == 0 { + return "" + } + if first := order[0].Ready; first != nil && (first.merging || !first.LastTry.IsZero() || first.RefusedSince != nil) { + return "" // the pass is under way, or its refusal is said on its own line + } + for _, m := range order { + if why := h.notHandable(m); why != "" { + return fmt.Sprintf("its group %s is not handed to the forge bridge: %s", g.ID, why) + } + } + return "" +} + +// noteNotHanded keeps on d's run why its approved merge is not handed, said once in the log when it changes, and as a +// stalled line meanwhile (readyLines). Under the lock. +func (h *Holder) noteNotHanded(d *Delivery, why string, now time.Time) { + r := d.Ready + if r == nil || r.NotHanded == why { + return + } + r.NotHanded = why + if why == "" { + r.NotHandedSince = nil + } else { + at := now + r.NotHandedSince = &at + h.Logf("[mesh-delivery] %s is approved, waits for nothing, and is NOT handed to merge: %s", d.ID, why) + } + h.keep(d) +} + // hand reads a group's required statuses and, when all are a success, writes the target as the member being merged. func (h *Holder) hand(gid, name, key, target string, reads []memberRead, approved map[string]string) { if gid != "" && !h.groupMergeable(gid, target, reads) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 94ffc5b8..3f16589d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -120,11 +120,13 @@ type ReadyRun struct { // in new words, so it never keeps a claim the check no longer holds. AskCheck string `json:"ask_check,omitempty"` // Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused - // why an answer was not taken (it did not hold to the ask). - Approved *time.Time `json:"approved,omitempty"` - Declined *time.Time `json:"declined,omitempty"` - Warrant string `json:"warrant,omitempty"` - Refused string `json:"refused,omitempty"` + // why an answer was not taken (it did not hold to the ask). AnsweredAsk is the ask the answer answers: an answer stands + // only for its own ask, so a Decline of an earlier ask never stands beside the Approve of a later one (declined). + Approved *time.Time `json:"approved,omitempty"` + Declined *time.Time `json:"declined,omitempty"` + AnsweredAsk string `json:"answered_ask,omitempty"` + Warrant string `json:"warrant,omitempty"` + Refused string `json:"refused,omitempty"` // One check per delivery (novox/hq ADR 0548 §2, §4, turns.go): HeldSince is when the approved merge began to wait for the // check of other heads HeldFor names (its verdict key), running on a repository it goes into; HandNow the operator's word @@ -137,6 +139,25 @@ type ReadyRun struct { // (writes.go, issue 1945); CannotWriteSince since when. Said at once as a condition. CannotWrite string `json:"cannot_write,omitempty"` CannotWriteSince *time.Time `json:"cannot_write_since,omitempty"` + + // NotHanded is why an approved merge that waits for nothing is not handed to the forge bridge (a member of its group that + // cannot be handed, and why), NotHandedSince since when: said once in the log and as a stalled line, never silent. + NotHanded string `json:"not_handed,omitempty"` + NotHandedSince *time.Time `json:"not_handed_since,omitempty"` +} + +// declined is whether the operator's Decline stands for this run: it answered the ask the run holds now, or the run holds +// no ask any more. A Decline of an earlier ask, the run asked again since at other heads (a sibling's new head), does not +// stand (2026-10-29: hq#495 declined under one ask and approved under the next was never handed). A run kept before answers +// named their ask stands declined unless an Approve came after the Decline. +func (r *ReadyRun) declined() bool { + if r == nil || r.Declined == nil { + return false + } + if r.AnsweredAsk != "" { + return r.AskID == "" || r.AskID == r.AnsweredAsk + } + return r.Approved == nil || r.Declined.After(*r.Approved) } // MergeBranch is the one branch this owner merges into (novox/hq ADR 0398 §4): a merge into any other is its author's. @@ -197,7 +218,7 @@ func (h *Holder) readyAsks(now time.Time) []func() { asks = append(asks, a) } r := d.Ready - if r == nil || r.Merged != nil || r.Declined != nil { + if r == nil || r.Merged != nil || r.declined() { continue } if d.State == Rejected { @@ -276,9 +297,11 @@ func (h *Holder) readyAsks(now time.Time) []func() { asks = append(asks, approvalAsk) } if waits != "" || r.merging { + h.noteNotHanded(d, "", now) // what it waits for is said by its waits, or it is handed continue } if !r.LastTry.IsZero() { + h.noteNotHanded(d, "", now) continue // refused: asked again at the next event that decides whether it may merge (mergeAgainNow), with no timer } // A group is merged one member at a time, handed to the forge bridge, and the pass stops at the first refusal (ADR @@ -288,7 +311,10 @@ func (h *Holder) readyAsks(now time.Time) []func() { g = nil } if a := h.handTurn(g, d, now); a != nil { + h.noteNotHanded(d, "", now) asks = append(asks, a) + } else { + h.noteNotHanded(d, h.notHanded(g, d), now) // a merge never handed says why (2026-10-29: silent for an hour) } } asks = append(asks, h.turnWrites()...) // the merge turns, after the holds of this look (ADR 0548 §3) @@ -575,6 +601,11 @@ func (h *Holder) readyLines(now time.Time) []StalledLine { Bound: "0s", H2: "none: a new head, which asks the operator again — the operator's", Says: fmt.Sprintf("%s#%d at %s is not merged: the operator's answer %s", d.Repository, d.Number, shortOf(r.Head, 12), r.Refused)}) + case r.NotHanded != "" && r.NotHandedSince != nil && r.Approved != nil: + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, + For: now.Sub(*r.NotHandedSince).Round(time.Second).String(), Bound: "0s", + H2: "none: the member named is approved at its head and ready, or its group is asked anew by a new head — the operator's", + Says: fmt.Sprintf("%s#%d at %s is approved and %s", d.Repository, d.Number, shortOf(r.Head, 12), r.NotHanded)}) case h.answering(d, now): // The trunk moved and its check was asked again: the gate is answering it, so it is no stall yet (ADR 0505 // §4). `show` lists the refusal meanwhile. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go index 72b3ab85..48a68aac 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go @@ -110,7 +110,7 @@ func (h *Holder) approvedMerges() []approvedMerge { for _, id := range ids { d := h.deliveries[id] r := d.Ready - if r == nil || r.Approved == nil || r.Declined != nil || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || + if r == nil || r.Approved == nil || r.declined() || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || d.State.Final() || d.State == Rejected || d.Base != MergeBranch || r.RefusedSince != nil || h.allMerged(r.Heads) { continue } @@ -382,7 +382,10 @@ type TurnLine struct { func (h *Holder) TurnsRead() map[string]any { h.mu.Lock() defer h.mu.Unlock() - want := h.wantedTurns() + want := map[string]sdk.Turn{} + if h.checkGates() { + want = h.wantedTurns() // as turnWrites: while the check gates no merge, no turn is written by design + } repos := make([]string, 0, len(want)) for repo := range want { repos = append(repos, repo) @@ -404,6 +407,9 @@ func (h *Holder) TurnsRead() map[string]any { lines = append(lines, l) } out := map[string]any{"turns": lines, "check-gates-merge": h.checkGates(), "bound": turnBound.String()} + if !h.checkGates() { + out["why-none"] = "the check gates no merge (check-gates-merge false): no merge waits for a check, so no turn is written" + } if h.Turns != nil { out["written"] = h.turnsLoaded var unwritten []string diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 57a6c550..3e3538db 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index 6245202d..a2b8e018 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-hand-acts/.gitignore b/modules/mesh-hand-acts/.gitignore new file mode 100644 index 00000000..2a60d0a0 --- /dev/null +++ b/modules/mesh-hand-acts/.gitignore @@ -0,0 +1,2 @@ +/mesh-hand-acts +/cmd/mesh-hand-acts/mesh-hand-acts diff --git a/modules/mesh-hand-acts/README.md b/modules/mesh-hand-acts/README.md new file mode 100644 index 00000000..b435e08d --- /dev/null +++ b/modules/mesh-hand-acts/README.md @@ -0,0 +1,26 @@ +# mesh-hand-acts + +Holds the seat `mesh-hand-acts` (novox/hq ADR 0565 §10, §11): the hand-act log, moved out of the controller. Go, one +binary (`cmd/mesh-hand-acts`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes the log. + +## What it keeps + +- the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream): + every act appended in its order under its key `act--` (`warrant--` for a warrant), in the layout + of the controller's log. The log keeps each act 400 days (`max-age-days`) and refuses a new act when + full rather than drop one; `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`. +- `mesh-hand-acts_warrants`: one key `.` per warrant, made once by create before its act is appended, so of + several instances one records it; a warrant whose act was lost is written again from the router's record. + +## Its verbs + +- `mesh-hand-acts.hand-act`: what, why, cause, the condition addressed, and the act's verb (a repair when not given; + `hand-act drill` for a drill, whose cause is always `drill`). Who did it is the caller the runtime proves. +- `mesh-hand-acts.warranted`: asker and ask. Read from the router's own record of that ask (`messenger.asks`), never + from the caller, and recorded once per ask; an ask still open, ended without a choice or another asker's is refused. +- `mesh-hand-acts.hand-acts`: both logs, this one and the controller's `mesh-controller_hand-acts` until its lease + ends, newest first, with the causes repaired by hand twice within a fortnight; `warrants=true` lists the warrants. + +No act is written to both logs: every recorder calls this seat from its delivery, and the controller keeps its own +acts in its own bucket until it loses its lease. diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go new file mode 100644 index 00000000..6e8579e8 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go @@ -0,0 +1,375 @@ +package main + +// The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a +// person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat +// mesh-hand-acts, once for the mesh, and is the one writer of its log `acts` — a stream, as ADR 0565 §10 has it: +// the mesh's module log (ADR 0297), append-only, every act kept in its order under its key, refusing a new act when +// full rather than dropping an old one, so every act is held 400 days and longer — and of its bucket `warrants` (one +// key `.` made once by create). Each act is said as `act-recorded`. +// +// **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the +// router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who +// chose, through which channel, with which proofs and which answer, once per ask. The controller's bucket +// `mesh-controller_hand-acts` stays its own until it loses its lease and is read, never written: `hand-acts` answers +// over both logs, and no act is written to both (ADR 0565 §11). + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// The seat, the log, the states and the event, as mesh-sdk's handacts names them. +const ( + Module = handacts.Module + Seat = handacts.Seat + ActsLog = handacts.ActsLog + WarrantsState = handacts.WarrantsState + ControllerActs = handacts.ControllerActs + // RouterRecords is the router's record of every ask, `.` (ADR 0259): the warrant is read there. + RouterRecords = "messenger.asks" + RecordedEvent = handacts.EventRecorded + // RecordedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + RecordedSubject = "mesh.seat." + Seat + ".event." + RecordedEvent + + VerbRecord = handacts.VerbRecord + VerbDrill = handacts.VerbDrill + VerbWarrant = handacts.VerbWarrant + CauseDrill = handacts.CauseDrill + CauseOperatorAnswer = handacts.CauseOperatorAnswer +) + +// RepeatedWithin is how far back a cause counts as repeated (S15). +const RepeatedWithin = 14 * 24 * time.Hour + +// Act is one entry of the log: mesh-sdk's handacts.Act, the controller's HandAct unchanged. +type Act = handacts.Act + +// Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only +// when it has none, and says ErrExists otherwise. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Create(state, key string, value any) error + Keys(state string) ([]string, error) + // Append adds an entry to the end of a log; Last is a key's newest entry; Entries every entry in order; Fill how + // many entries and bytes the log holds against its cap. + Append(log, key string, value any) error + Last(log, key string) ([]byte, bool, error) + Entries(log string) ([][]byte, error) + Fill(log string) (Fill, error) +} + +// Fill is how full a log is: its entries, its bytes and its cap in bytes. +type Fill struct { + Entries uint64 `json:"entries"` + Bytes uint64 `json:"bytes"` + MaxBytes uint64 `json:"max-bytes"` +} + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Log is mesh-hand-acts: the log and what records into it. +type Log struct { + Store Store + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) +} + +var seq atomic.Uint64 + +// actKey is an act's key: its time and a sequence, so the keys list in order, as the controller's do. +func actKey(at time.Time) string { + return "act-" + strconv.FormatInt(at.UnixNano(), 10) + "-" + strconv.FormatUint(seq.Add(1), 10) +} + +// Recorded is what a recorder is answered: the act as kept. +type Recorded struct { + Act Act `json:"act"` + Said string `json:"said"` +} + +// HandAct records one act a caller says it did, by hand or on purpose: what, why, its cause and the condition it +// addresses. by is the caller the runtime proved. verb is the act's verb as the log names it: a repair by default, a +// drill with VerbDrill, or a recorder's own (`conditions silence`, `config set`), which S15 reads as it reads the +// controller's. +func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, error) { + what, why, cause, verb = strings.TrimSpace(what), strings.TrimSpace(why), strings.TrimSpace(cause), strings.TrimSpace(verb) + switch { + case !proven(by): + return Recorded{}, fmt.Errorf("who acts is the caller the runtime proves, and it proved none (%q): nothing was recorded", by) + case what == "": + return Recorded{}, errors.New("a hand act says what was done: nothing was recorded") + case why == "": + return Recorded{}, errors.New("a hand act says why: nothing was recorded") + } + if verb == "" { + verb = VerbRecord + } + if verb == VerbDrill { + cause = CauseDrill + } + switch { + case cause == "": + return Recorded{}, errors.New("a hand act says its cause, in a word or a condition's kind: the word a second act " + + "for the same reason uses, which is how a repair done twice is found. Nothing was recorded") + case strings.EqualFold(cause, CauseDrill) && verb != VerbDrill: + return Recorded{}, errors.New("a drill is not recorded as a repair: record it with verb " + VerbDrill + + ", which no healer is wanted for. Nothing was recorded") + } + at := l.Now().UTC() + act := Act{ID: actKey(at), At: at, By: by, Verb: verb, Args: []string{what}, Why: why, Cause: cause, + Condition: strings.TrimSpace(condition)} + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s did %q, because %q (cause: %s)", act.ID, by, what, + why, cause)}, nil +} + +// record writes one act and says it. An act not said is still kept, and the log says so. +func (l *Log) record(act Act) error { + if err := l.Store.Append(ActsLog, act.ID, act); err != nil { + return fmt.Errorf("the act could not be recorded: %w", err) + } + if l.Publish != nil { + if err := l.Publish(RecordedSubject, act, act.ID); err != nil { + l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err) + } + } + return nil +} + +// routerRecord is what warranted reads of the router's record of an ask. +type routerRecord struct { + State string `json:"state"` + Warrant *asks.Warrant `json:"warrant"` + Ask map[string]any `json:"ask"` +} + +var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// Warrant is one ask's warrant as recorded: the key `.` of `warrants`, naming the act it recorded. +type Warrant struct { + Act string `json:"act"` + At time.Time `json:"at"` + By string `json:"recorded-at-the-word-of"` +} + +// Warranted records an asker's act on the operator's warrant for its ask, read from the router's record, once per ask +// however often it is asked (ADR 0274, ADR 0259). The caller gives which ask and nothing else. +func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) { + switch { + case !proven(caller): + return Recorded{}, fmt.Errorf("who asks is the caller the runtime proves, and it proved none (%q): nothing was recorded", caller) + case !askerModule.MatchString(asker): + return Recorded{}, fmt.Errorf("%q is not a module's name", asker) + case !asks.UsableID(ask): + return Recorded{}, fmt.Errorf("%q is not an ask's id", ask) + } + key := asker + "." + ask + claimed, found, err := l.Store.Get(WarrantsState, key) + if err != nil { + return Recorded{}, err + } + if found { + if r, ok, err := l.already(key, claimed); err != nil || ok { + return r, err + } + } + raw, found, err := l.Store.Get(RouterRecords, key) + if err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err) + } + var rec routerRecord + if found { + if err := json.Unmarshal(raw, &rec); err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s is not readable: %w", asker, ask, err) + } + } + w := rec.Warrant + switch { + case rec.State == "open" || rec.State == "held": + return Recorded{}, fmt.Errorf("%s's ask %s is still %s: nobody has answered it. Nothing was recorded", asker, ask, rec.State) + case !found || rec.State == "" || w == nil: + return Recorded{}, fmt.Errorf("the router holds no closed record of %s's ask %s. Nothing was recorded", asker, ask) + case w.Asker != asker || w.Ask != ask: + return Recorded{}, fmt.Errorf("the router's record is for %s's ask %s. Nothing was recorded", w.Asker, w.Ask) + case w.Outcome != asks.OutcomeChosen || w.By == nil: + return Recorded{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record. Nothing was "+ + "recorded", asker, ask, w.Outcome) + case w.AskDigest == "": + return Recorded{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest. Nothing was recorded", asker, ask) + } + at := l.Now().UTC() + act := Act{ID: "warrant-" + asker + "-" + ask, At: w.At.UTC(), Verb: VerbWarrant, + Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)}, + Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker), + By: fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity), + Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs, + RequestedBy: asker + ", recorded at the word of " + caller, + Outcome: "chosen; what " + asker + " did with it is in its own record"} + if claimed == nil { + // The claim first: of any number of instances asking at once, the one whose create lands records the act. + if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) { + return Recorded{Said: "another instance records " + act.ID + " now"}, nil + } else if err != nil { + return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err) + } + } + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil +} + +// already answers a warrant recorded before, when its act is in the log; false when the act was lost between the +// claim and the log, so it is read from the router again and written. +func (l *Log) already(key string, raw []byte) (Recorded, bool, error) { + var w Warrant + if err := json.Unmarshal(raw, &w); err != nil { + return Recorded{}, false, fmt.Errorf("the warrant %s is not readable: %w", key, err) + } + actRaw, found, err := l.Store.Last(ActsLog, w.Act) + if err != nil || !found { + return Recorded{}, false, err + } + var act Act + _ = json.Unmarshal(actRaw, &act) + return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, true, nil +} + +func via(w asks.Warrant) string { + if w.By == nil { + return w.Channel + } + v := w.Channel + " (" + w.By.Kind + ")" + if w.By.Verified != "" { + v += ", " + w.By.Verified + } + return v +} + +// proven is a bus user the runtime proved: never empty, never the SDK's unproven word. +func proven(by string) bool { + by = strings.TrimSpace(by) + return by != "" && !strings.EqualFold(by, "unproven") && !strings.HasPrefix(by, "(") +} + +// Listing is what hand-acts answers: every act of both logs within the days asked, newest first, narrowed by cause +// and who, and the causes repaired by hand twice or more within a fortnight. +type Listing struct { + Acts []Act `json:"acts"` + Repeated map[string]int `json:"repeated"` + Days int `json:"days"` + // Unread says which log could not be read, so what is missing is said and never read as nothing. + Unread map[string]string `json:"unread,omitempty"` + Kept map[string]int `json:"kept"` + // Fill is how full this seat's log is against its cap: full, it refuses the next act. + Fill *Fill `json:"fill,omitempty"` +} + +// HandActs reads both logs. +func (l *Log) HandActs(days int, cause, who string) Listing { + now := l.Now() + since := now.Add(-time.Duration(days) * 24 * time.Hour) + out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}} + var all []Act + for _, log := range []string{ActsLog, ControllerActs} { + acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin))) + if err != nil { + if out.Unread == nil { + out.Unread = map[string]string{} + } + out.Unread[log] = err.Error() + continue + } + out.Kept[log] = kept + all = append(all, acts...) + } + for _, a := range all { + if now.Sub(a.At) <= RepeatedWithin && a.Repair() { + out.Repeated[a.Cause]++ + } + if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) { + continue + } + out.Acts = append(out.Acts, a) + } + for c, n := range out.Repeated { + if n < 2 { + delete(out.Repeated, c) + } + } + sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) }) + if f, err := l.Store.Fill(ActsLog); err == nil { + out.Fill = &f + } + return out +} + +// read is every act of one log since a moment, and how many it keeps: this seat's log entry by entry, the +// controller's bucket by its keys, each of which names its time, so only those within are read whole. +func (l *Log) read(log string, since time.Time) ([]Act, int, error) { + var out []Act + keep := func(raw []byte, from string) { + var a Act + if json.Unmarshal(raw, &a) != nil || a.At.Before(since) { + return + } + a.Log = from + out = append(out, a) + } + if log == ActsLog { + entries, err := l.Store.Entries(ActsLog) + if err != nil { + return nil, 0, err + } + for _, raw := range entries { + keep(raw, handacts.LogSeat) + } + return out, len(entries), nil + } + keys, err := l.Store.Keys(log) + if err != nil { + return nil, 0, err + } + for _, k := range keys { + if at, ok := keyTime(k); ok && at.Before(since) { + continue + } + raw, found, err := l.Store.Get(log, k) + if err != nil { + return nil, len(keys), err + } + if found { + keep(raw, handacts.LogController) + } + } + return out, len(keys), nil +} + +// keyTime is the time an act's key names (`act--`), when it names one. +func keyTime(key string) (time.Time, bool) { + parts := strings.Split(key, "-") + if len(parts) != 3 || parts[0] != "act" { + return time.Time{}, false + } + ns, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(0, ns), true +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go new file mode 100644 index 00000000..73d42cef --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "sync" + "testing" + "time" +) + +type entry struct { + key string + raw []byte +} + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + logs map[string][]entry + keysErr map[string]error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, logs: map[string][]entry{}, keysErr: map[string]error{}} +} + +func (s *memStore) Append(log, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return err + } + s.logs[log] = append(s.logs[log], entry{key, raw}) + return nil +} + +func (s *memStore) Last(log, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i := len(s.logs[log]) - 1; i >= 0; i-- { + if s.logs[log][i].key == key { + return s.logs[log][i].raw, true, nil + } + } + return nil, false, nil +} + +func (s *memStore) Entries(log string) ([][]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return nil, err + } + var out [][]byte + for _, e := range s.logs[log] { + out = append(out, e.raw) + } + return out, nil +} + +func (s *memStore) Fill(log string) (Fill, error) { + s.mu.Lock() + defer s.mu.Unlock() + return Fill{Entries: uint64(len(s.logs[log])), MaxBytes: 1 << 30}, nil +} + +// logKeys are a log's keys in order. +func (s *memStore) logKeys(log string) []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for _, e := range s.logs[log] { + out = append(out, e.key) + } + return out +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func newLog() (*Log, *memStore, *[]Act, *time.Time) { + store := newStore() + var said []Act + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now }, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(Act); ok && subject == "mesh.seat.mesh-hand-acts.event.act-recorded" && id == a.ID { + said = append(said, a) + } + return nil + }} + return l, store, &said, &now +} + +// hand-act records what the proven caller says it did, once, and says it; it refuses what cannot be read back, a +// caller the runtime did not prove, and a drill recorded as a repair (ADR 0565 §10; issue 1953 L5-C1). +func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) { + cases := []struct { + name, by, verb, what, why, cause string + wantVerb, wantCause, err string + }{ + {"a repair", "ace.mesh-config", "", "set dns", "it was wrong", "config", VerbRecord, "config", ""}, + {"a recorder's own verb", "novox.mesh-conditions", "conditions silence", "silenced x", "known", "operator-answer", + "conditions silence", "operator-answer", ""}, + {"a drill takes its cause", "novox.mesh-healers", VerbDrill, "stopped records", "does it raise", "", VerbDrill, CauseDrill, ""}, + {"no proven caller", "unproven", "", "x", "y", "z", "", "", "proves"}, + {"by said empty", "", "", "x", "y", "z", "", "", "proves"}, + {"no why", "person.jochen", "", "x", "", "z", "", "", "says why"}, + {"no what", "person.jochen", "", "", "y", "z", "", "", "says what"}, + {"no cause", "person.jochen", "", "x", "y", "", "", "", "cause"}, + {"a drill as a repair", "person.jochen", "", "x", "y", "drill", "", "", "not recorded as a repair"}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing") + keys := store.logKeys(ActsLog) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || len(keys) != 1 || keys[0] != r.Act.ID || len(*said) != 1 { + t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said)) + } + var kept Act + raw, _, _ := store.Last(ActsLog, keys[0]) + _ = json.Unmarshal(raw, &kept) + if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" || + kept.Args[0] != tc.what || kept.Log != "" { + t.Errorf("%s: kept %+v", tc.name, kept) + } + } +} + +// warranted records the router's word once per ask, never the caller's; it refuses an ask still open, one ended +// without a choice, another asker's, and one the router holds no record of (ADR 0565 §10, ADR 0259). +func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) { + chosen := `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"chosen","label":"Approve", + "by":{"who":"operator","kind":"telegram","identity":"42","verified":"P2"},"channel":"telegram","proofs":["P1","P2"], + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}` + cases := []struct { + name, record, asker, ask, err string + // claimed is a warrant claimed by an instance that stopped before its act reached the log. + claimed bool + }{ + {"chosen", chosen, "claude-code", "a1", "", false}, + {"claimed, its act lost: written now", chosen, "claude-code", "a1", "", true}, + {"still open", `{"state":"open"}`, "claude-code", "a1", "still open", false}, + {"no record", "", "claude-code", "a1", "no closed record", false}, + {"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`, + "claude-code", "a1", "no person chose", false}, + {"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1", + "record is for mesh-config", false}, + {"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest", false}, + {"not a module", chosen, "Claude Code", "a1", "not a module", false}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + if tc.record != "" { + _ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record)) + } + if tc.claimed { + _ = store.Create(WarrantsState, "claude-code.a1", Warrant{Act: "warrant-claude-code-a1"}) + } + r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask) + if tc.err != "" { + if keys := store.logKeys(ActsLog); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || r.Act.Verb != VerbWarrant || r.Act.Cause != CauseOperatorAnswer || + !strings.Contains(r.Act.By, "operator, as telegram identity 42") || r.Act.Via != "telegram (telegram), P2" || + !strings.Contains(r.Act.RequestedBy, "ace.claude-code") || len(r.Act.Proofs) != 2 { + t.Fatalf("%s: %+v, %v", tc.name, r.Act, err) + } + // Asked again, by another instance: nothing more is recorded. + again, err := l.Warranted("g14.claude-code", "claude-code", "a1") + keys := store.logKeys(ActsLog) + if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 { + t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said)) + } + } +} + +// hand-acts answers over both logs, newest first, narrowed by days, cause and who, and counts each cause repaired by +// hand twice in a fortnight across both, never a person's decision; a log it cannot read is said, never read as empty. +func TestHandActsReadsBothLogs(t *testing.T) { + l, store, _, now := newLog() + put := func(log, verb, cause, by string, ago time.Duration) { + at := now.Add(-ago) + a := Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"} + if log == ActsLog { + _ = store.Append(log, actKey(at), a) + return + } + _ = store.Put(log, actKey(at), a) + } + put(ActsLog, VerbRecord, "relogin", "person.jochen", time.Hour) + put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour) + put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour) + put(ActsLog, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour) + put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour) + cases := []struct { + name string + days int + cause, who string + want int + }{ + {"a fortnight", 14, "", "", 6}, + {"a day", 1, "", "", 5}, + {"forty days", 40, "", "", 7}, + {"by cause", 14, "relogin", "", 2}, + {"by who", 14, "", "claude-code", 1}, + } + for _, tc := range cases { + got := l.HandActs(tc.days, tc.cause, tc.who) + if len(got.Acts) != tc.want || len(got.Repeated) != 1 || got.Repeated["relogin"] != 2 || got.Unread != nil { + t.Errorf("%s: %d acts, repeated %v, unread %v", tc.name, len(got.Acts), got.Repeated, got.Unread) + } + for i := 1; i < len(got.Acts); i++ { + if got.Acts[i].At.After(got.Acts[i-1].At) { + t.Errorf("%s: not newest first", tc.name) + } + } + } + if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 || got.Fill == nil || + got.Fill.Entries != 4 { + t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept) + } + store.keysErr[ControllerActs] = errors.New("the bus is away") + if got := l.HandActs(14, "", ""); got.Unread[ControllerActs] == "" || len(got.Acts) != 4 { + t.Errorf("an unreadable log: %+v", got.Unread) + } + w, err := l.Warrants() + if err != nil || w.(map[string]any)["count"] != 0 { + t.Errorf("warrants answered %v, %v", w, err) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go new file mode 100644 index 00000000..b103fa23 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go @@ -0,0 +1,84 @@ +// mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, its module +// log: a stream that keeps every act in order) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for +// `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Create(state, key string, value any) error { + _, err := stdio.State(state).Create(key, value) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeStore) Append(log, key string, value any) error { + _, err := stdio.Log(log).Append(key, value) + return err +} + +func (runtimeStore) Last(log, key string) ([]byte, bool, error) { + e, err := stdio.Log(log).Last(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Entries(log string) ([][]byte, error) { + var out [][]byte + for from := uint64(0); ; { + page, err := stdio.Log(log).Read(from, "", 1000) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, e.Value) + } + if page.Next == 0 { + return out, nil + } + from = page.Next + } +} + +func (runtimeStore) Fill(log string) (Fill, error) { + i, err := stdio.Log(log).Info() + if err != nil { + return Fill{}, err + } + return Fill{Entries: i.Entries, Bytes: i.Bytes, MaxBytes: i.MaxBytes}, nil +} + +func main() { + l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf, + Publish: stdio.SeatPublish} + if err := stdio.Serve("", Tools(l)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go new file mode 100644 index 00000000..cd19669a --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565 +// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, keeps its +// acts as its log (a stream, §10) and warrants as a bucket that never ages, so a warrant is recorded once. Every state +// and log it keeps and reads has a tool that reads it (ADR 0565 §15). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Runs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + Logs []struct { + Name string `json:"name"` + MaxAgeDays int `json:"max-age-days"` + } `json:"logs"` + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.Runs != Module { + t.Errorf("module %q runs as %q", m.Module, m.Runs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v", m.Claims) + } + // act-recorded is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { + t.Errorf("emits %v, reads %v", m.Emits, m.Reads) + } + readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, + RouterRecords: WarrantedTool} + tools := map[string]bool{} + for _, tool := range Tools(&Log{}) { + tools[tool.Name] = true + } + // The acts are a log kept 400 days (ADR 0565 §10). + if len(m.Logs) != 1 || m.Logs[0].Name != ActsLog || m.Logs[0].MaxAgeDays != 400 { + t.Errorf("logs %+v, want the acts as a log kept 400 days", m.Logs) + } + for _, raw := range m.State { + var name string + if json.Unmarshal(raw, &name) != nil || name != WarrantsState { + t.Errorf("the state %s is not the warrants, or ages", raw) + } + if !tools[readBy[name]] { + t.Errorf("the state %s has no tool that reads it", name) + } + } + if !tools[readBy[ActsLog]] { + t.Errorf("the log %s has no tool that reads it", ActsLog) + } + for _, r := range m.Reads { + if !tools[readBy[r]] { + t.Errorf("the read %s has no tool that answers it", r) + } + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside new file mode 100644 index 00000000..60d6785c --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-hand-acts/module.json diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go new file mode 100644 index 00000000..833db625 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go @@ -0,0 +1,110 @@ +package main + +// The seat's verbs (novox/hq ADR 0565 §10, §15): hand-act and warranted record, hand-acts reads both logs and, with +// warrants, every warrant recorded. A recovery is a call: a warrant whose act was lost is written again by asking +// warranted again, from the router's record. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"hand-act", "hand-acts", "warranted"} + +// Tool names. +const ( + HandActTool = Seat + ".hand-act" + HandActsTool = Seat + ".hand-acts" + WarrantedTool = Seat + ".warranted" +) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +// Tools are what the runtime serves. +func Tools(l *Log) []stdio.Tool { + return []stdio.Tool{ + {Name: HandActTool, + Description: "Record one act done by hand (novox/hq ADR 0565 §10): what was done, why, its cause in a word or a " + + "condition's kind (the word a second act for the same reason uses, which is how a repair done twice is " + + "found) and the condition it addresses. Who did it is the caller the runtime proves. verb is the act's " + + "verb as the log names it: a repair when not given, `hand-act drill` for a drill, or a recorder's own " + + "(`conditions silence`, `config set`). Said as act-recorded.", + Input: obj(map[string]any{"what": str("what was done"), "why": str("why, in a sentence"), + "cause": str("the cause, in a word or a condition's kind"), "condition": str("the condition's key, if any"), + "verb": str("the act's verb in the log; a repair when not given")}, "what", "why"), + Run: func(a map[string]any) (any, error) { + return l.HandAct(stdio.CallerOf(a), arg(a, "verb"), arg(a, "what"), arg(a, "why"), arg(a, "cause"), arg(a, "condition")) + }}, + {Name: HandActsTool, + Description: "What was done by hand lately, over both logs (this seat's and the controller's until its lease " + + "ends), newest first: each act with who, why, its cause and which log holds it, and the causes repaired by " + + "hand twice or more within a fortnight (a healer is wanted). Narrowed by cause and who; warrants=true lists " + + "every warrant recorded instead.", + Input: obj(map[string]any{"days": map[string]any{"type": "number", "description": "how many days back (default 14)"}, + "cause": str("only this cause"), "who": str("only acts whose who contains this"), + "warrants": str("\"true\": every warrant recorded, `.` with its act")}), + Run: func(a map[string]any) (any, error) { + if strings.EqualFold(arg(a, "warrants"), "true") || a["warrants"] == true { + return l.Warrants() + } + return l.HandActs(days(a), arg(a, "cause"), arg(a, "who")), nil + }}, + {Name: WarrantedTool, + Description: "Record an asker's act on the operator's warrant (ADR 0274, ADR 0259): read from the router's own " + + "record of that ask — who chose, through which channel, with which proofs and which answer — never from the " + + "caller, once per ask; refused for an ask still open, ended without a choice, or another asker's.", + Input: obj(map[string]any{"asker": str("the module that asked"), "ask": str("its ask's id")}, "asker", "ask"), + Run: func(a map[string]any) (any, error) { + return l.Warranted(stdio.CallerOf(a), arg(a, "asker"), arg(a, "ask")) + }}, + } +} + +// Warrants is every warrant recorded, by key. +func (l *Log) Warrants() (any, error) { + keys, err := l.Store.Keys(WarrantsState) + if err != nil { + return nil, fmt.Errorf("the warrants cannot be read: %w", err) + } + sort.Strings(keys) + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := l.Store.Get(WarrantsState, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{"warrants": rows, "count": len(rows)}, nil +} + +func arg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func days(a map[string]any) int { + switch v := a["days"].(type) { + case float64: + if v > 0 { + return int(v) + } + case string: + var n int + if _, err := fmt.Sscan(v, &n); err == nil && n > 0 { + return n + } + } + return 14 +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go new file mode 100644 index 00000000..64ec46d2 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -0,0 +1,405 @@ +//go:build integration + +package main + +// mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered +// from a real nats-server's key-value buckets, mesh/log.* from the log's real stream, mesh/publish onto its subjects). A recorder's hand act is kept in the +// log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info": + p.log(m) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +// log answers the runtime's log verbs from the log's real stream, LOG__ on mesh.log.... +func (p *player) log(m wire) { + var a struct { + Log string `json:"log"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + From uint64 `json:"from"` + } + _ = json.Unmarshal(m.Params, &a) + stream := "LOG_" + Module + "_" + a.Log + subject := "mesh.log." + Module + "." + a.Log + s, err := p.js.Stream(p.ctx, stream) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entry := func(r *jetstream.RawStreamMsg) map[string]any { + return map[string]any{"sequence": r.Sequence, "key": strings.TrimPrefix(r.Subject, subject+"."), + "value": json.RawMessage(r.Data), "at": r.Time} + } + switch m.Method { + case "mesh/log.append": + ack, err := p.js.Publish(p.ctx, subject+"."+a.Key, a.Value) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"sequence": ack.Sequence}, nil) + case "mesh/log.last": + r, err := s.GetLastMsgForSubject(p.ctx, subject+"."+a.Key) + if errors.Is(err, jetstream.ErrMsgNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, entry(r), nil) + case "mesh/log.read": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entries := []map[string]any{} + for seq := max(a.From, 1); seq <= info.State.LastSeq; seq++ { + if r, err := s.GetMsg(p.ctx, seq); err == nil { + entries = append(entries, entry(r)) + } + } + p.answer(m.ID, map[string]any{"entries": entries, "next": 0, "last": info.State.LastSeq}, nil) + case "mesh/log.info": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"entries": info.State.Msgs, "bytes": info.State.Bytes, + "max_bytes": info.Config.MaxBytes, "first": info.State.FirstSeq, "last": info.State.LastSeq}, nil) + } +} + +func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1953/L5-C2: an act recorded through the seat is kept on the bus and heard as act-recorded. +func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-hand-acts does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + // The log as the mesh makes it (ADR 0297): a stream kept by limits, refusing when full, never deleting. + logStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_" + Module + "_" + ActsLog, + Subjects: []string{"mesh.log." + Module + "." + ActsLog + ".>"}, MaxBytes: 64 << 20, Discard: jetstream.DiscardNew, + DenyDelete: true, DenyPurge: true, AllowDirect: true}) + if err != nil { + t.Fatal(err) + } + recorded, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + RecordedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(HandActTool, "ace.mesh-config", map[string]any{"what": "set dns on g14", "why": "it was wrong", + "cause": "config", "by": "someone else"}) + if err != nil { + t.Fatal(err) + } + var r Recorded + if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" { + t.Fatalf("hand-act answered %s", raw) + } + e, err := logStream.GetLastMsgForSubject(ctx, "mesh.log."+Module+"."+ActsLog+"."+r.Act.ID) + if err != nil || !strings.Contains(string(e.Data), `"by":"ace.mesh-config"`) { + t.Fatalf("the act is not kept in the log on the bus: %v", err) + } + msg, err := recorded.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) { + t.Fatalf("act-recorded was not said: %v", err) + } + // A warrant the router recorded, asked twice: one act. + asks, _ := js.KeyValue(ctx, "messenger_asks") + if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code", + "outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram", + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil { + t.Fatal(err) + } + for _, caller := range []string{"ace.claude-code", "g14.claude-code"} { + if _, err := p.call(WarrantedTool, caller, map[string]any{"asker": "claude-code", "ask": "a1"}); err != nil { + t.Fatal(err) + } + } + if info, err := logStream.Info(ctx); err != nil || info.State.Msgs != 2 { + t.Errorf("the log keeps %v, want the act and one warrant (%v)", info, err) + } + listed, err := p.call(HandActsTool, "person.jochen", map[string]any{}) + if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) { + t.Errorf("hand-acts answered %s, %v", listed, err) + } +} diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod new file mode 100644 index 00000000..412f5438 --- /dev/null +++ b/modules/mesh-hand-acts/go.mod @@ -0,0 +1,22 @@ +module mesh-hand-acts + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-hand-acts/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json new file mode 100644 index 00000000..96aff8b1 --- /dev/null +++ b/modules/mesh-hand-acts/module.json @@ -0,0 +1,65 @@ +{ + "module": "mesh-hand-acts", + "version": "1", + "runs-as": "mesh-hand-acts", + "claims": [ + { + "name": "mesh-hand-acts", + "scope": "mesh", + "serves": [ + "hand-act", + "hand-acts", + "warranted" + ] + } + ], + "reads": [ + "mesh-controller.hand-acts", + "messenger.asks" + ], + "logs": [ + { + "name": "acts", + "max-age-days": 400 + } + ], + "state": [ + "warrants" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-hand-acts", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-hand-acts", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-hand-acts" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-hand-acts", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-hand-acts", + "binary": "mesh-hand-acts", + "loads": [ + "mesh-hand-acts" + ] + } + ] + } +} diff --git a/modules/mesh-healers/.gitignore b/modules/mesh-healers/.gitignore new file mode 100644 index 00000000..0bafa572 --- /dev/null +++ b/modules/mesh-healers/.gitignore @@ -0,0 +1,2 @@ +/mesh-healers +/cmd/mesh-healers/mesh-healers diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md new file mode 100644 index 00000000..d00cdb1b --- /dev/null +++ b/modules/mesh-healers/README.md @@ -0,0 +1,46 @@ +# mesh-healers + +Holds the seat `mesh-healers` (novox/hq ADR 0564 §7, §8): the mesh's healers, moved out of the controller. Go, one +binary (`cmd/mesh-healers`), served by a runtime of its own on its own bus account (`runs-as`). + +## What it does + +It hears the condition keeper's `condition-raised` and `condition-changed` (seat `mesh-conditions`) on a durable +consumer. A condition whose kind a healer answers is acted on within the healer's budget, its settle and the mesh-wide +brake (twelve acts an hour). A repair is a call of the seat or tool that owns the act: H3 asks the grants part's +`grants_assert` for the module whose consumer is gone. H4 (the controller's own consumer) and H5 (the provider's own +repair) are registered and run elsewhere. + +Each act is begun by a create of its key in `attempts` before it is made and finished after, so a holder that dies +mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's +`tried`, resolver and escalation from `attempts`; a healer writes no condition. + +**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would +take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. + +Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared +`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits +are keys of `wakes` whose time to live is the wait: `settle.` ends with a settle and judges the condition +again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears +each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while +no instance watched is judged at the start, once the lease is first read. + +S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a +fortnight; a drill and a person's decision by design never count. + +## What it keeps + +- `mesh-healers_attempts`: every act and escalation, each key kept a week. +- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease, + each key kept a week. +- `mesh-healers_findings`: the brake, its own blindness, and S15. +- `mesh-healers_wakes`: the waits, each key living as long as it waits. + +## Its verbs and tools + +- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why; + `recount=true` counts the hand-act log again. +- `mesh-healers.heal`: one act now on an open condition, by key and why. +- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the + controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill. +- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go new file mode 100644 index 00000000..425eba47 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -0,0 +1,663 @@ +package main + +// The healers (novox/hq ADR 0564 §7, §8; ADR 0231): a known failure heals itself, under a brake, and every repair is +// said. The registry is compiled in; mesh-healers hears `condition-raised` and `condition-changed` of the seat +// mesh-conditions on a durable consumer, instead of looking every half minute. A repair is a call of the seat or tool +// that owns the act (H3: the grants part's grants_assert). Each act is begun by a create of its key in the bucket +// `attempts` before it is made and finished after, so a holder that dies mid-act has spent it and a second instance +// cannot begin the same act; budgets, settles and the mesh-wide brake count from there. Success is never the +// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`, +// the resolver and the escalation into the condition from `attempts`. +// +// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is +// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches +// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once +// the lease is handed away nothing is written there again. + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-healers" + Seat = "mesh-healers" +) + +// The states it keeps and reads, and the event it says. +const ( + AttemptsState = health.AttemptsState + WouldState = health.WouldState + FindingsState = health.FindingsState + LeaseRead = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey + ConditionsRead = "mesh-conditions.conditions" + HandActsRead = "mesh-controller.hand-acts" + ActedEvent = health.EventHealerActed + // ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays). + ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent + // RecordedEvent is the hand-act log's seat event, each act recorded there (ADR 0565 §11). + RecordedEvent = handacts.Seat + "." + handacts.EventRecorded + // The events it hears: the keeper's, on the seat mesh-conditions. + RaisedEvent = "mesh-conditions.condition-raised" + ChangedEvent = "mesh-conditions.condition-changed" + // AssertTool is the grants part's tool that asserts a module's buckets, logs and streams again. + AssertTool = "mesh-grants.grants_assert" +) + +// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour. +const ( + BrakeLimit = 12 + BrakeWindow = time.Hour +) + +// The outcomes an attempt has. +const ( + OutcomeActing = health.OutcomeActing + OutcomeActed = health.OutcomeActed + OutcomeFailed = health.OutcomeFailed + OutcomeEscalated = health.OutcomeEscalated +) + +// Store is the bus's buckets as this module reaches them, through its runtime. CreateTTL writes a key only when it has +// none, and says ErrExists otherwise; it and PutTTL give the key its own time to live, at whose end the bus removes it +// and a watch of the bucket hears it expire (a bucket declared key-ttl). +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + CreateTTL(state, key string, value any, ttl time.Duration) error + PutTTL(state, key string, value any, ttl time.Duration) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// KeptFor is how long an attempt and a would-act are kept: each key a week from its last write (ADR 0564 Limits). +const KeptFor = 7 * 24 * time.Hour + +// WakesState holds the healers' waits as keys whose time to live is the wait: `settle.` ends with a +// settle, `fortnight` when S15's oldest counted act leaves the fortnight. Its expiry, heard on the bucket's watch, is +// the wake: no timer runs in the process, and a restart loses nothing a key on the bus holds. +const WakesState = "wakes" + +// The wakes' keys. +const ( + settlePrefix = "settle." + fortnightKey = "fortnight" +) + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside. +type Condition struct { + Key string `json:"key"` + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity string `json:"severity"` + Resolver string `json:"resolver"` + Tried []json.RawMessage `json:"tried,omitempty"` +} + +// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until +// observation clears it. +func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 } + +// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key +// `..` is the healers' own), an act written to `would` while the lease names a +// controller (one key per healer and condition, `.`, its newest), and the body of healer-acted, +// epoch left out (ADR 0564 Limits 10). +type ( + Attempt = health.Attempt + Would = health.Would + Acted = health.HealerActed +) + +// Healer is one row of the registry. +type Healer struct { + ID string + Kinds []string + Condition string + Repair string + Then string + From string + Budget int + Window time.Duration + Settle time.Duration + // ActsIn is where the repair runs: here, or a module that repairs its own (H5), or the controller (H4). + ActsIn string + // applies says whether this condition is one it may act on, and what its budget counts against; why when not. + applies func(c Condition) (budget string, ok bool, why string) + // repair is the act: what it did, what came of it, or why it could not. + repair func(h *Healers, c Condition) (act, said string, err error) +} + +// actsHere marks a healer whose repair mesh-healers makes. +const actsHere = "mesh-healers" + +// Registry is to-be 45 §7's table, compiled in: every healer the mesh has, the ones mesh-healers runs with their act. +func Registry() []Healer { + return []Healer{ + {ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"}, + Condition: "a durable consumer of a module the bus does not hold (D6), or a seat's holder that does not answer (D3)", + Repair: "assert the module's buckets, logs and streams again through the grants part (grants_assert)", + Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)", + Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsHere, + applies: appliesToAModulesObject, repair: assertAgain}, + {ID: "H4", Kinds: []string{"consumer-behind"}, + Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable", + Repair: "re-make the consumer to deliver from now; the one marked resettable is the controller's own", + Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)", + Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, + ActsIn: "the controller, until its lease: the one consumer marked resettable is its own (ADR 0564 §7)"}, + {ID: "H5", Kinds: []string{"provider-failing"}, + Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing)", + Repair: "the provider repairs it itself through the server's own bootstrap command (ADR 0224 §5)", + Then: "announced failing by the provider, braked from ten minutes doubling to six hours", + From: "the identity provider's admin reset through its bootstrap command (179)", + Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute, + ActsIn: "the provider module (keycloak), ADR 0224 §5"}, + } +} + +// Healers is mesh-healers: the registry, what its acts reach, and what it holds. +type Healers struct { + Store Store + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) + Registry []Healer + + mu sync.Mutex + // lease is the controller's lease as last read: acting is on only while it is handed away. + acting bool + leaseWhy string + // declined is why each condition was last passed over, said once. + declined map[string]string + // acts are the hand-act log as watched, for S15. + acts map[string]HandAct +} + +// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564 +// §8, Limits 8, as ADR 0561 §3's reader reads it). +func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) { + on, why := buscompose.LeaseHandedAway(raw, found, readErr) + h.mu.Lock() + flipped := on != h.acting || h.leaseWhy == "" + h.acting, h.leaseWhy = on, leaseWords(on, why) + h.mu.Unlock() + if flipped { + h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy) + } +} + +// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting. +func leaseWords(on bool, why string) string { + if on { + return "the controller's lease is handed away: the healers act" + } + if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") { + return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would" + } + return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would" +} + +// Acting says whether the healers act, and why. +func (h *Healers) Acting() (bool, string) { + h.mu.Lock() + defer h.mu.Unlock() + if h.leaseWhy == "" { + return false, "the controller's lease has not been read yet: every act is written as would" + } + return h.acting, h.leaseWhy +} + +// healerFor is the healer that acts here on a kind. +func (h *Healers) healerFor(kind string) (Healer, bool) { + for _, r := range h.Registry { + if r.repair != nil && slices.Contains(r.Kinds, kind) { + return r, true + } + } + return Healer{}, false +} + +// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over. +func (h *Healers) Heard(body []byte) error { + var c Condition + if err := json.Unmarshal(body, &c); err != nil { + h.Logf("[%s] a condition event is not readable, and is passed over: %v", Module, err) + return nil + } + _, err := h.Judge(c, "") + return err +} + +// Judge is one condition judged by the healer registered for its kind. by names a person's `heal`, empty for an +// event. It answers what was done, in words. +func (h *Healers) Judge(c Condition, by string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + row, ok := h.healerFor(c.Kind) + if !ok { + return "no healer here answers the kind " + c.Kind, nil + } + if c.Escalated() { + return "it is handed to the operator already: no healer touches it until observation clears it", nil + } + budget, applies, why := row.applies(c) + if !applies { + h.decline(c.Key, row.ID, why) + return fmt.Sprintf("healer %s leaves it alone: %s", row.ID, why), nil + } + delete(h.declined, c.Key) + now := h.Now() + if !h.acting { + w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()} + if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil { + return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err) + } + return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil + } + all, err := h.attempts() + if err != nil { + h.blind(err) + return "", fmt.Errorf("the healers cannot read what they did, so they count no budget and act on nothing: %w", err) + } + h.see("blind", nil) + acts := actsWithin(all, now.Add(-BrakeWindow)) + if braked, said := h.brakeHolds(acts, now); braked { + h.see("brake", []Observation{brakeFinding(acts, said)}) + return "every healer has stopped: " + said, nil + } + h.see("brake", nil) + spent := spentOn(all, row, budget, now) + if n := len(spent); n > 0 && now.Sub(spent[n-1].Begun) < row.Settle { + h.wake(budget, spent[n-1].Begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s waits: its last act on it is still the observation's to judge until %s", row.ID, + spent[n-1].Begun.Add(row.Settle).UTC().Format(time.RFC3339)), nil + } + if len(spent) >= row.Budget { + return h.escalate(row, c, budget, spent) + } + if len(acts) >= BrakeLimit { + return "the brake holds from the next act on", nil + } + return h.act(row, c, budget, len(spent), by) +} + +func (h *Healers) whyNot() string { + if h.leaseWhy == "" { + return "the controller's lease has not been read yet" + } + return h.leaseWhy +} + +// act is one healer's act: begun by a create in attempts, made, finished, and said. +func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) { + begun := h.Now().UTC() + key := attemptKey(row.ID, c.Key, begun) + if by == "" { + by = "healer " + row.ID + } + window := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: row.Repair, Outcome: OutcomeActing, + Begun: begun} + if by != "healer "+row.ID { + a.Said = "asked by " + by + } + if err := h.Store.CreateTTL(AttemptsState, key, a, KeptFor); err != nil { + if errors.Is(err, ErrExists) { + return "another instance began this act first", nil + } + return "", fmt.Errorf("healer %s did not act on %s: its attempt could not be begun: %w", row.ID, c.Key, err) + } + act, said, err := row.repair(h, c) + a.Outcome = OutcomeActed + if err != nil { + a.Outcome, said = OutcomeFailed, err.Error() + } + if act != "" { + a.Act = act + } + finished := h.Now().UTC() + if a.Said != "" { + said = a.Said + "; " + said + } + a.Said, a.Finished = said+" ("+window+")", &finished + if err := h.Store.PutTTL(AttemptsState, key, a, KeptFor); err != nil { + h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window}) + h.wake(budget, begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s %s %s: %s — %s (%s)", row.ID, a.Outcome, c.Key, a.Act, said, window), nil +} + +// escalate is a spent budget: the condition is the operator's now. Written once as an attempt the keeper turns into +// the condition's resolver and urgency (ADR 0564 §7); an escalated condition is passed over until it clears. +func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attempt) (string, error) { + var tried []string + for _, s := range spent { + tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.Begun.UTC().Format("15:04 MST"), firstLine(s.Said))) + } + said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window, + c.Key, strings.Join(tried, "; ")) + now := h.Now().UTC() + window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator", + Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now} + if err := h.Store.CreateTTL(AttemptsState, attemptKey(row.ID, c.Key, now), a, KeptFor); err != nil && !errors.Is(err, ErrExists) { + return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated, + Said: said, Budget: window}) + return fmt.Sprintf("healer %s handed %s to the operator: %s", row.ID, c.Key, said), nil +} + +// tell says a heal as healer-acted. Not said is said in the log: the attempt holds it. +func (h *Healers) tell(e Acted) { + e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer + e.Show = "mesh-conditions.conditions key=" + e.Condition + if h.Publish == nil { + return + } + if err := h.Publish(ActedSubject, e, e.Healer+"."+e.Condition+"."+e.Outcome+"."+strconv.FormatInt(e.At.UnixNano(), 10)); err != nil { + h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err) + } +} + +// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has +// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still +// running keeps its key on the bus and needs nothing. +func (h *Healers) Rearm() { + all, err := h.attempts() + if err != nil { + h.Logf("[%s] what the healers did cannot be read, so no ended settle is judged: %v", Module, err) + return + } + now := h.Now() + newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first + for _, a := range all { + if a.Outcome != OutcomeEscalated { + newest[a.Healer+" "+a.BudgetKey] = a + } + } + var due []string + for _, a := range newest { + for _, r := range h.Registry { + if r.ID == a.Healer && r.repair != nil && !now.Before(a.Begun.Add(r.Settle)) && now.Sub(a.Begun) <= r.Window { + due = append(due, a.Condition) + } + } + } + sort.Strings(due) + for _, key := range due { + h.again(key) + } +} + +// wake writes the wait for a settle's end as a key whose time to live is the wait; its expiry judges the condition +// again (WakeHeard). Written again with the same end, it is the same wait. +func (h *Healers) wake(_ string, at time.Time, key string) { + ttl := at.Sub(h.Now()).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, settlePrefix+key, map[string]any{"condition": key, "at": at.UTC()}, ttl); err != nil { + h.Logf("[%s] the settle of %s is not written as a wait, so its end is judged at the next event: %v", Module, key, err) + } +} + +// WakeHeard is a change of the wakes bucket: an expiry is a wait ended — a settle's, whose condition is judged again, +// or S15's fortnight, counted again. +func (h *Healers) WakeHeard(key, op string) { + if op != OpExpire { + return + } + switch { + case key == fortnightKey: + h.Wanted() + case strings.HasPrefix(key, settlePrefix): + h.again(strings.TrimPrefix(key, settlePrefix)) + } +} + +// OpExpire is the watch's word for a key whose own time to live ended. +const OpExpire = stdio.OpExpire + +// again judges a condition again from the keeper's bucket, if it is still open. +func (h *Healers) again(key string) { + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil || !found { + return + } + if _, err := h.Judge(conditionOf(raw), ""); err != nil { + h.Logf("[%s] %s judged again at its settle's end: %v", Module, key, err) + } +} + +func conditionOf(raw []byte) Condition { + var c Condition + _ = json.Unmarshal(raw, &c) + return c +} + +// decline remembers why a healer passed a condition over, said once. +func (h *Healers) decline(key, healer, why string) { + if h.declined == nil { + h.declined = map[string]string{} + } + if h.declined[key] != why { + h.Logf("[%s] healer %s leaves %s alone: %s", Module, healer, key, why) + } + h.declined[key] = why +} + +func attemptKey(healer, condition string, begun time.Time) string { + return healer + "." + condition + "." + strconv.FormatInt(begun.UnixNano(), 10) +} + +// attempts are every attempt kept, oldest first. +func (h *Healers) attempts() ([]Attempt, error) { + keys, err := h.Store.Keys(AttemptsState) + if err != nil { + return nil, err + } + var out []Attempt + for _, k := range keys { + raw, found, err := h.Store.Get(AttemptsState, k) + if err != nil { + return nil, err + } + if !found { + continue + } + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the attempt %s is not readable: %w", k, err) + } + out = append(out, a) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Begun.Before(out[j].Begun) }) + return out, nil +} + +// actsWithin are the attempts since a moment that were acts: an escalation is a saying, and the brake does not count it. +func actsWithin(all []Attempt, since time.Time) []Attempt { + var out []Attempt + for _, a := range all { + if !a.Begun.Before(since) && a.Outcome != OutcomeEscalated { + out = append(out, a) + } + } + return out +} + +// spentOn is one healer's acts against one budget key within its window, oldest first. +func spentOn(all []Attempt, row Healer, budget string, now time.Time) []Attempt { + var out []Attempt + for _, a := range actsWithin(all, now.Add(-row.Window)) { + if a.Healer == row.ID && a.BudgetKey == budget { + out = append(out, a) + } + } + return out +} + +// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake already written +// and an act within the hour still — it lets go an hour after the last act, not the first. +func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) { + said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), BrakeWindow, BrakeLimit) + if len(acts) >= BrakeLimit { + return true, said + } + _, held, _ := h.Store.Get(FindingsState, brakeKey) + if held && len(acts) > 0 { + last := acts[len(acts)-1].Begun + return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said, + last.Add(BrakeWindow).UTC().Format("15:04 MST")) + } + return false, said +} + +var brakeKey = health.Key(ScopeMesh, "healers", "braked") + +// brakeFinding is the brake, as the urgent finding that says it. +func brakeFinding(acts []Attempt, said string) Observation { + counts := map[string]int{} + for _, a := range acts { + counts[a.Healer+" on "+a.Condition]++ + } + var most []string + for what, n := range counts { + most = append(most, fmt.Sprintf("%s ×%d", what, n)) + } + sort.Strings(most) + return Observation{Kind: "healers-braked", Token: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"}, + Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " + + "healer looping, not repairing — " + said, + Said: strings.Join(most, ", "), + Headline: "All automatic repairs stopped", + Needs: "read the details and decide whether the repairs may run again.", + Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " + + "like a loop, so all of them stopped.", + Resolved: "Automatic repairs run again"} +} + +// blind writes the healers' own blindness: they cannot read what they did. +func (h *Healers) blind(err error) { + h.see("blind", []Observation{{Kind: "probe-failed", Token: "failed", Severity: Warning, + Subject: Subject{Scope: ScopeProbe, ID: "healers"}, + Summary: "the healers cannot read what they did, so they count no budget and act on nothing", + Said: firstLine(err.Error()), Headline: "A check of the mesh could not run", + Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.", + Resolved: "The automatic repairs read what they did again"}}) +} + +// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it +// no longer sees (ADR 0564 §2). +func (h *Healers) see(source string, found []Observation) { + now := h.Now() + want := map[string]health.Finding{} + for _, o := range found { + o.Source = source + want[o.Key()] = o.Finding("", now) + } + keys, err := h.Store.Keys(FindingsState) + if err != nil { + h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err) + return + } + for _, k := range keys { + raw, found, err := h.Store.Get(FindingsState, k) + if err != nil || !found { + continue + } + was, err := health.DecodeFinding(raw) + if err != nil || was.Source != source { + continue + } + if f, still := want[k]; still { + if says(was) == says(f) { + delete(want, k) + } + continue + } + if err := h.Store.Delete(FindingsState, k); err != nil { + h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err) + } + } + for k, f := range want { + raw, err := health.EncodeFinding(f) + if err == nil { + err = h.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + h.Logf("[%s] the finding %s is not written: %v", Module, k, err) + } + } +} + +// --- H3 -------------------------------------------------------------------------------------------------------- + +// moduleOf is the module whose objects a condition names: a consumer of a module's own (`.`). The +// controller's own consumers are not a module's: the controller asserts them itself until its lease. +func moduleOf(c Condition) (string, string) { + stream, consumer, ok := strings.Cut(c.Subject.ID, ".") + if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" { + return "", "it names no consumer of a module whose objects the grants part asserts" + } + if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") { + return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease" + } + return consumer, "" +} + +// appliesToAModulesObject is H3's: a module's consumer the bus does not hold, its budget per condition. +func appliesToAModulesObject(c Condition) (string, bool, string) { + if _, why := moduleOf(c); why != "" { + return "", false, why + } + return c.Key, true, "" +} + +// assertAgain is H3: the grants part asserts the module's buckets, logs and streams again. +func assertAgain(h *Healers, c Condition) (string, string, error) { + module, why := moduleOf(c) + if module == "" { + return "", "", errors.New(why) + } + raw, err := h.Ask(AssertTool, map[string]any{"module": module}) + if err != nil { + return "", "", fmt.Errorf("the grants part did not assert %s: %w", module, err) + } + var a struct { + Says string `json:"says"` + } + _ = json.Unmarshal(raw, &a) + return "asserted " + module + "'s buckets, logs and streams again through the grants part", + orSaid(a.Says, "the grants part answered") + "; the probe that raised it says on its next run whether it is there", nil +} + +func orSaid(s, otherwise string) string { + if strings.TrimSpace(s) == "" { + return otherwise + } + return s +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go new file mode 100644 index 00000000..9b6f9dce --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -0,0 +1,496 @@ +package main + +import ( + "encoding/json" + "errors" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" +) + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + keysErr map[string]error + ttl map[string]time.Duration +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +// CreateTTL and PutTTL keep the key's time to live beside it, for a test to read. +func (s *memStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, true); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) PutTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, false); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) setTTL(state, key string, ttl time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ttl == nil { + s.ttl = map[string]time.Duration{} + } + s.ttl[state+"/"+key] = ttl +} + +// ttlOf is a key's time to live as last written; zero for none. +func (s *memStore) ttlOf(state, key string) time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return s.ttl[state+"/"+key] +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) keys(state string) []string { + k, _ := s.Keys(state) + return k +} + +func (s *memStore) attempt(t *testing.T, key string) Attempt { + t.Helper() + raw, _, _ := s.Get(AttemptsState, key) + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatalf("%s: %s", key, raw) + } + return a +} + +// world is mesh-healers over memory, with its clock, its asks, what it said and the wakes it set. +type world struct { + h *Healers + store *memStore + now time.Time + asked []string + said []Acted +} + +func newWorld() *world { + w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)} + w.fresh() + return w +} + +// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten. +func (w *world) fresh() { + w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, + Now: func() time.Time { return w.now }, + Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + w.asked = append(w.asked, key+" "+string(raw)) + return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil + }, + Publish: func(subject string, body any, _ string) error { + if a, ok := body.(Acted); ok && subject == "mesh.seat.mesh-healers.event.healer-acted" { + w.said = append(w.said, a) + } + return nil + }, + } +} + +func (w *world) handAway() { + w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) +} + +// lost is a module's consumer the bus does not hold, as D6 raises it. +func lost(module string) Condition { + return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self", + Subject: Subject{Scope: "bus", ID: "EVENTS." + module}} +} + +func body(c Condition) []byte { + raw, _ := json.Marshal(map[string]any{"key": c.Key, "kind": c.Kind, "subject": c.Subject, "severity": c.Severity, + "resolver": c.Resolver, "tried": c.Tried, "event": "condition-raised", "show": "mesh-conditions.conditions key=" + c.Key}) + return raw +} + +// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no +// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8). +func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) { + cases := []struct { + name string + lease func(h *Healers) + acts bool + }{ + {"never read", func(*Healers) {}, false}, + {"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false}, + {"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false}, + {"a controller holds it", func(h *Healers) { + h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil) + }, false}, + {"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true}, + } + for _, tc := range cases { + w := newWorld() + tc.lease(w.h) + if err := w.h.Heard(body(lost("records"))); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState) + if tc.acts { + if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) || + len(w.said) != 1 || w.said[0].Outcome != OutcomeActed { + t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said) + continue + } + a := w.store.attempt(t, attempts[0]) + if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" { + t.Errorf("%s: the attempt is %+v", tc.name, a) + } + continue + } + if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) { + t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said) + } + if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked { + t.Errorf("%s: a would-act moved the brake", tc.name) + } + } +} + +// An act waits out its settle, then its spent budget hands the condition to the operator once; an escalated condition +// is passed over; a second instance cannot begin an act already begun; the brake stops every healer past twelve acts +// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7). +func TestBudgetSettleEscalationAndBrake(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + settle := settlePrefix + c.Key + steps := []struct { + name string + after time.Duration + cond Condition + attempts int + outcome string // the newest said outcome, empty for none new + // how is what happens: an event, a restart over the same buckets, or the settle key's expiry heard. + how string + // wait is the settle key's time to live after the step, zero for no check. + wait time.Duration + }{ + {"acts: its settle is a key that expires", 0, c, 1, OutcomeActed, "event", 6*time.Minute + time.Second}, + {"within its settle: waits for the same end", 2 * time.Minute, c, 1, "", "event", 4*time.Minute + time.Second}, + {"a restart within its settle: the key on the bus still waits", time.Minute, c, 1, "", "restart", 4*time.Minute + time.Second}, + {"its key expires: judged again, budget spent, escalates", 4 * time.Minute, c, 2, OutcomeEscalated, "expire", 0}, + {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", "event", 0}, + } + for _, s := range steps { + w.now = w.now.Add(s.after) + before := len(w.said) + switch s.how { + case "restart": + w.fresh() + w.handAway() + w.h.Rearm() + case "expire": + w.h.WakeHeard(settle, OpExpire) + default: + if _, err := w.h.Judge(s.cond, ""); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + } + if n := len(w.store.keys(AttemptsState)); n != s.attempts { + t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts) + } + if s.wait != 0 && w.store.ttlOf(WakesState, settle) != s.wait { + t.Errorf("%s: the settle waits %s, want %s", s.name, w.store.ttlOf(WakesState, settle), s.wait) + } + switch { + case s.outcome == "" && len(w.said) != before: + t.Errorf("%s: said %+v", s.name, w.said[before:]) + case s.outcome != "" && (len(w.said) != before+1 || w.said[before].Outcome != s.outcome): + t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) + } + } + // Each attempt is kept a week from its last write. + for _, k := range w.store.keys(AttemptsState) { + if ttl := w.store.ttlOf(AttemptsState, k); ttl != KeptFor { + t.Errorf("the attempt %s is kept %s, want a week", k, ttl) + } + } + // A settle whose key expired while no instance watched is judged at the start. + r := newWorld() + r.handAway() + _ = r.store.Put(ConditionsRead, c.Key, c) + if _, err := r.h.Judge(c, ""); err != nil { + t.Fatal(err) + } + r.now = r.now.Add(7 * time.Minute) + r.fresh() + r.handAway() + r.h.Rearm() + if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated { + t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said) + } + // A second instance cannot begin the act an attempt's key already holds. + x := newWorld() + x.handAway() + _ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor) + if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 { + t.Errorf("a second instance acted: %q, asked %v", said, x.asked) + } + // The brake: twelve acts in the hour stop every healer, said as an urgent finding. + b := newWorld() + b.handAway() + for i := 0; i < BrakeLimit; i++ { + b.now = b.now.Add(time.Minute) + _ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x", + Outcome: OutcomeActed, Begun: b.now}, KeptFor) + } + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 { + t.Errorf("past the brake: %q, asked %v", said, b.asked) + } + raw, braked, _ := b.store.Get(FindingsState, brakeKey) + f, _ := health.DecodeFinding(raw) + if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module { + t.Errorf("the brake is written as %s", raw) + } + b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") { + t.Errorf("the brake let go before an hour after the last act: %q", said) + } + b.now = b.now.Add(6 * time.Minute) + if _, err := b.h.Judge(lost("mail"), ""); err != nil || len(b.asked) != 1 { + t.Errorf("an hour after the last act the healers do not act again: %v, asked %v", err, b.asked) + } + if _, braked, _ := b.store.Get(FindingsState, brakeKey); braked { + t.Error("the brake's finding outlived the brake") + } + // Blind to what they did, they act on nothing and say so. + z := newWorld() + z.handAway() + z.store.keysErr[AttemptsState] = errors.New("the bus is away") + if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 { + t.Errorf("blind, the healers acted: %v", err) + } + if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) { + t.Errorf("blind, the findings are %v", keys) + } +} + +// H3 answers a module's consumer the bus does not hold, never the controller's own or a condition naming no consumer; +// the registry holds every healer the mesh has, H3 run here, H4 and H5 run elsewhere (ADR 0231 §2, ADR 0564 §7). +func TestTheRegistryAndWhatH3Answers(t *testing.T) { + cases := []struct { + c Condition + holds bool + }{ + {lost("records"), true}, + {lost("controller"), false}, + {lost("controller-merges"), false}, + {Condition{Key: "seat.x.silent", Kind: "holder-silent", Subject: Subject{Scope: "seat", ID: "issue-tracker"}}, false}, + {Condition{Key: "bus.EVENTS.records.behind", Kind: "consumer-behind", Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}, false}, + } + for _, tc := range cases { + w := newWorld() + w.handAway() + said, _ := w.h.Judge(tc.c, "") + if acted := len(w.asked) == 1; acted != tc.holds { + t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said) + } + } + runs := map[string]bool{} + for _, r := range Registry() { + if len(r.Kinds) == 0 || r.Budget < 1 || r.Window <= 0 || r.Settle <= 0 || r.Repair == "" || r.ActsIn == "" { + t.Errorf("%s is not a whole row: %+v", r.ID, r) + } + runs[r.ID] = r.repair != nil + } + if !runs["H3"] || runs["H4"] || runs["H5"] || len(runs) != 3 { + t.Errorf("the healers run here are %v, want H3 alone of H3, H4, H5", runs) + } +} + +// S15: a cause repaired by hand twice within a fortnight is a finding; a decision by design never counts; it clears +// when fewer than two remain, and the healers wake when the oldest leaves the fortnight (ADR 0564 Limits 2). +func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { + w := newWorld() + act := func(key, verb, cause string, ago time.Duration) { + raw, _ := json.Marshal(HandAct{ID: key, At: w.now.Add(-ago), By: "person.jochen", Verb: verb, Why: "x", Cause: cause}) + w.h.HandActHeard(key, "put", raw) + } + act("a1", "broker consumer-reset", "consumer-behind", 10*24*time.Hour) + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Fatalf("one act is a finding: %v", keys) + } + // The second of the cause is the seat's own act, heard as act-recorded: one in each log counts two (ADR 0565 §11). + seatAct, _ := json.Marshal(HandAct{ID: "a2", At: w.now.Add(-24 * time.Hour), By: "person.jochen", + Verb: "broker consumer-reset", Why: "x", Cause: "consumer-behind"}) + w.h.ActRecorded(seatAct) + // A drill in each log counts none. + act("a3", "hand-act drill", "drill", time.Hour) + drill, _ := json.Marshal(HandAct{ID: "a4", At: w.now.Add(-2 * time.Hour), Verb: "hand-act drill", Why: "x", Cause: "drill"}) + w.h.ActRecorded(drill) + act("a5", "conditions silence", "operator-answer", time.Hour) + act("a6", "conditions silence", "operator-answer", time.Hour) + act("a7", "hand-act record", "relogin", 20*24*time.Hour) + act("a8", "hand-act record", "relogin", time.Hour) + want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"} + if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) { + t.Fatalf("findings %v, want %v", keys, want) + } + raw, _, _ := w.store.Get(FindingsState, want[0]) + if !strings.Contains(string(raw), "healer H4 answers this cause") { + t.Errorf("the finding does not say its healer is not enough: %s", raw) + } + if ttl := w.store.ttlOf(WakesState, fortnightKey); ttl != 4*24*time.Hour+time.Second { + t.Errorf("the fortnight waits %s, want the oldest act's leaving it in 4 days", ttl) + } + w.now = w.now.Add(4*24*time.Hour + time.Second) + w.h.Wanted() + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Errorf("past the fortnight the finding stays: %v", keys) + } + w.h.HandActHeard("a5", "delete", nil) + if len(w.h.acts) != 7 { + t.Errorf("a deleted entry is kept: %d", len(w.h.acts)) + } +} + +// The verbs: heal acts on an open condition by key and says why it was asked; drill waits for the log's home; the reads +// answer every bucket. +func TestTheVerbsAndReads(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil { + t.Error("heal took no why") + } + if _, err := w.h.Heal("bus.none", "x", "person.jochen"); err == nil { + t.Error("heal acted on no open condition") + } + if _, err := w.h.Heal(c.Key, "it is gone again", "person.jochen"); err != nil { + t.Fatal(err) + } + keys := w.store.keys(AttemptsState) + if len(keys) != 1 || !strings.Contains(w.store.attempt(t, keys[0]).Said, "asked by person.jochen (it is gone again)") { + t.Errorf("heal's attempt is %v", keys) + } + answer, err := w.h.Answer(7, true) + if err != nil { + t.Fatal(err) + } + a := answer.(map[string]any) + if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 { + t.Errorf("healers answered %v", a) + } + // drill records into mesh-hand-acts as a drill; while that seat has no holder, into the controller's drill until + // the lease is handed away (ADR 0565 §11). + drills := []struct { + name string + seatGone bool + leaseAway bool + args map[string]any + log, err string + }{ + {"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""}, + {"no holder: the controller's", true, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogController, ""}, + {"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "handed its lease away"}, + {"says nothing", false, true, map[string]any{}, "", "says what"}, + } + for _, d := range drills { + x := newWorld() + if d.leaseAway { + x.handAway() + } else { + x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil) + } + var asked []string + x.h.Ask = func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if d.seatGone && strings.HasPrefix(key, "seat:mesh-hand-acts.") { + return nil, errors.New("nats: no responders available for request") + } + return json.RawMessage(`{}`), nil + } + var got any + var err error + for _, tool := range Tools(x.h) { + if tool.Name == DrillTool { + got, err = tool.Run(d.args) + } + } + switch { + case d.err != "": + if err == nil || !strings.Contains(err.Error(), d.err) { + t.Errorf("drill %s: %v", d.name, err) + } + case err != nil || got.(map[string]any)["log"] != d.log: + t.Errorf("drill %s: %v, %v", d.name, got, err) + case d.log == handacts.LogSeat && !strings.Contains(asked[0], `"verb":"hand-act drill"`): + t.Errorf("drill %s asked %v", d.name, asked) + } + } + if _, err := w.h.read(WouldState, ""); err != nil { + t.Error(err) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go new file mode 100644 index 00000000..6e48bfba --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -0,0 +1,118 @@ +// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its +// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun, +// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds +// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's +// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and watches the +// controller's hand-act log for S15. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) PutTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).PutTTL(key, value, ttl) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state, key string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch(key, handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + +func main() { + h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} + // The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended + // while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry. + var started sync.Once + go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { + h.LeaseHeard(c.Value, c.Op == "put", nil) + started.Do(func() { go h.Rearm() }) + return nil + }) + go watched(WakesState, "", func(c stdio.StateChange) error { + h.WakeHeard(c.Key, c.Op) + return nil + }) + go watched(HandActsRead, "", func(c stdio.StateChange) error { + h.HandActHeard(c.Key, c.Op, c.Value) + return nil + }) + // S15 over both logs: the seat's act-recorded on a durable consumer, beside the controller's bucket's watch. + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(RecordedEvent, func(e stdio.Envelope) error { + h.ActRecorded(e.Body) + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, RecordedEvent, err, wait) + } + }() + for _, event := range []string{RaisedEvent, ChangedEvent} { + go func(event string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(event, func(e stdio.Envelope) error { return h.Heard(e.Body) }) + if err == nil { + logf("[%s] hears %s", Module, event) + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, event, err, wait) + } + }(event) + } + if err := stdio.Serve("", Tools(h)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go new file mode 100644 index 00000000..af3745b9 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1, +// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's +// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each. +// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + for _, c := range []struct { + name string + got, want []string + }{ + {"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil}, + {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}}, + {"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-controller.drill"}}, + {"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}}, + } { + if !slices.Equal(c.got, c.want) { + t.Errorf("%s %v, want %v", c.name, c.got, c.want) + } + } + states := map[string]bool{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + KeyTTL bool `json:"key-ttl"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.KeyTTL + } + // Each key of attempts and would lives a week from its last write (KeptFor, would has no other deleter), and each + // wait in wakes as long as it waits: their buckets take a key's own time to live. + if !states[AttemptsState] || !states[WouldState] || !states[WakesState] || states[FindingsState] { + t.Errorf("the states take a key's own time to live as %v, want attempts, would and wakes alone", states) + } + readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool, + WakesState: WakesTool, "a settle's end": HealTool, + "an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool} + tools := map[string]bool{} + for _, tool := range Tools(&Healers{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + if name, _, _ := strings.Cut(tool, " "); !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/observation.go b/modules/mesh-healers/cmd/mesh-healers/observation.go new file mode 100644 index 00000000..853550b5 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/observation.go @@ -0,0 +1,77 @@ +package main + +// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their +// blindness and S15, each written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the +// controller's key for the same condition. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is what saw it: the brake, the healers' blindness, or S15. + Source string +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} diff --git a/modules/mesh-healers/cmd/mesh-healers/reads-outside b/modules/mesh-healers/cmd/mesh-healers/reads-outside new file mode 100644 index 00000000..cdc54f27 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-healers/module.json diff --git a/modules/mesh-healers/cmd/mesh-healers/tools.go b/modules/mesh-healers/cmd/mesh-healers/tools.go new file mode 100644 index 00000000..d3452b95 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/tools.go @@ -0,0 +1,230 @@ +package main + +// The seat's verbs and the module's own reads (novox/hq ADR 0564 §7, §12): `healers` is the registry, what the healers +// did and would do, the brake and whether they act; `heal` takes one act now on a condition, within budget and brake; +// `drill` waits for the hand-act log's home; and a read of each bucket it keeps, so a recovery is a call. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"healers", "heal", "drill"} + +// Tool names. +const ( + HealersTool = Seat + ".healers" + HealTool = Seat + ".heal" + DrillTool = Seat + ".drill" + FindingsTool = "healers_findings" + WouldTool = "healers_would" + WakesTool = "healers_wakes" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(h *Healers) []stdio.Tool { + return []stdio.Tool{ + {Name: HealersTool, + Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " + + "days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " + + "would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " + + "again for S15 (a cause repaired by hand twice in a fortnight).", + Input: obj(map[string]any{ + "days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"}, + "recount": map[string]any{"type": "boolean", "description": "count the hand-act log again now"}}), + Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }}, + {Name: HealTool, + Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " + + "event would have it — or, while the controller holds its lease, the act it would take written as would. " + + "Says what was done.", + Input: obj(map[string]any{ + "key": map[string]any{"type": "string", "description": "the condition's key"}, + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}), + Run: func(a map[string]any) (any, error) { + return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a)) + }}, + {Name: DrillTool, + Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with what it " + + "tests, in the hand-act log of the seat mesh-hand-acts (the controller's drill while that seat has no " + + "holder and the controller holds its lease). A drill is never a repair: S15 never counts it.", + Input: obj(map[string]any{"what": map[string]any{"type": "string", "description": "what was done on purpose"}, + "why": map[string]any{"type": "string", "description": "what the drill tests"}, + "condition": map[string]any{"type": "string", "description": "the condition it is meant to raise, if any"}}), + Run: func(a map[string]any) (any, error) { + return h.Drill(strArg(a, "what"), strArg(a, "why"), strArg(a, "condition")) + }}, + {Name: FindingsTool, + Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " + + "each cause repaired by hand twice in a fortnight (S15); or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }}, + {Name: WakesTool, + Description: "What the healers wait for (their bucket wakes): each settle still running (`settle.`) " + + "and S15's fortnight end (`fortnight`), each a key whose time to live is the wait and whose expiry wakes them; " + + "or one whole by key. heal acts on a condition now, and healers with recount=true counts S15 now.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }}, + {Name: WouldTool, + Description: "What the healers would have done while the controller holds its lease (their bucket would, each " + + "key kept a week): one per healer and condition, its newest; or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }}, + } +} + +// Drill records a drill (ADR 0565 §11, which ends ADR 0564 Limits 1's refusal): through the seat mesh-hand-acts as +// the act `hand-act drill`, and while that seat has no holder that answers and the controller still holds its lease, +// through the controller's own `drill`, which records it as a drill too. It answers which log took it. +func (h *Healers) Drill(what, why, condition string) (any, error) { + if what == "" || why == "" { + return nil, errors.New("a drill says what was done on purpose and what it tests: nothing was recorded") + } + if h.Ask == nil { + return nil, errors.New("the hand-act log cannot be reached from here: nothing was recorded") + } + body := map[string]any{"what": what, "why": why, "verb": handacts.VerbDrill} + controller := map[string]any{"what": what, "why": why} + if condition != "" { + body["condition"], controller["condition"] = condition, condition + } + answer, err := h.Ask("seat:"+handacts.Seat+".hand-act", body) + if err == nil { + return map[string]any{"log": handacts.LogSeat, "answer": answer}, nil + } + if !handacts.NoResponder(err) { + return nil, fmt.Errorf("%s.hand-act refused the drill: %w", handacts.Seat, err) + } + if acting, _ := h.Acting(); acting { + return nil, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing was "+ + "recorded (%w)", handacts.Seat, err) + } + answer, cerr := h.Ask("seat:mesh-controller.drill", controller) + if cerr != nil { + return nil, fmt.Errorf("%s has no holder that answers (%v), and the controller's drill refused too: %w", handacts.Seat, + err, cerr) + } + return map[string]any{"log": handacts.LogController, "answer": answer}, nil +} + +// Heal is the verb heal: one condition judged now by its healer. +func (h *Healers) Heal(key, why, caller string) (any, error) { + if key == "" || why == "" { + return nil, errors.New("heal names the condition's key and says why") + } + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil { + return nil, fmt.Errorf("the open conditions cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("no condition is open under %s", key) + } + said, err := h.Judge(conditionOf(raw), caller+" ("+why+")") + if err != nil { + return nil, err + } + return map[string]any{"said": said}, nil +} + +// Answer is the verb healers. +func (h *Healers) Answer(days int, recount bool) (any, error) { + if recount { + h.Wanted() + } + now := h.Now() + all, err := h.attempts() + if err != nil { + return nil, fmt.Errorf("what the healers did cannot be read: %w", err) + } + var recent []Attempt + for _, a := range all { + if now.Sub(a.Begun) <= time.Duration(days)*24*time.Hour { + recent = append(recent, a) + } + } + sort.SliceStable(recent, func(i, j int) bool { return recent[i].Begun.After(recent[j].Begun) }) + braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now) + var rows []map[string]any + for _, r := range h.Registry { + rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair, + "budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle), + "then": r.Then, "acts-in": r.ActsIn, "from": r.From}) + } + acting, why := h.Acting() + would, err := h.read(WouldState, "") + if err != nil { + would = map[string]any{"unread": err.Error()} + } + h.mu.Lock() + declined := map[string]string{} + for k, v := range h.declined { + declined[k] = v + } + h.mu.Unlock() + if recent == nil { + recent = []Attempt{} + } + return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would, + "brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()}, + "declined": declined, + "note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil +} + +// read is every key of a bucket with its value, or one whole. +func (h *Healers) read(state, key string) (any, error) { + if key != "" { + raw, found, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("nothing is kept under %s in %s", key, state) + } + return json.RawMessage(raw), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := h.Store.Get(state, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{state: rows, "count": len(rows)}, nil +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go new file mode 100644 index 00000000..065f2c41 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -0,0 +1,166 @@ +package main + +// S15: a cause repaired by hand twice within a fortnight is a healer wanted (ADR 0231 §27, §28; ADR 0564 Limits 2). +// The hand-act log is the controller's bucket mesh-controller_hand-acts, a key-value bucket with no events: mesh-healers +// watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the +// finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design +// (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the +// fortnight: the key `fortnight` of the healers' wakes expires when the oldest counted act leaves it. From the log's home (issue 1953) they +// watch that instead. + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// Within is how far back a repeated cause counts. +const Within = 14 * 24 * time.Hour + +// HandAct is what S15 reads of one entry of the controller's hand-act log (mesh-controller internal/link HandAct). +type HandAct struct { + ID string `json:"id"` + At time.Time `json:"at"` + By string `json:"by"` + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` + Why string `json:"why"` + Cause string `json:"cause"` + Kind string `json:"kind,omitempty"` +} + +// decided are the verbs whose entries are a person's decision by design, with the causes they hold for (every cause +// when none): the controller's handActVerbs, read as S15 reads them. A verb not listed counts as a repair. +var decided = map[string][]string{ + "warrant": nil, + "seat": {"seat-passed"}, + "conditions silence": {"operator-answer"}, + "hand-act record": {"drill"}, + "secret issued": nil, + "hand-act drill": nil, + "retire approve": nil, + "retire reject": nil, + "cleanup delete": nil, + "dead-letters deliver": nil, + "dead-letters drop": nil, + "collect": nil, + "mirrors": nil, + "move-placement": nil, + "move-config": nil, + "bus upgrade": nil, + "upgrade release-backlog": nil, + "secret accept": nil, + "secret rotate": {"leaked-in-logs"}, + "rotate": {"leaked-in-logs"}, +} + +// personsDecision is whether an act is a person's decision by design, by the verb that recorded it. +func personsDecision(a HandAct) bool { + causes, ok := decided[a.Verb] + if !ok { + return false + } + return len(causes) == 0 || slices.Contains(causes, a.Cause) +} + +// ActRecorded is one act the seat mesh-hand-acts recorded, heard as act-recorded: counted beside the controller's +// log's, under a key of its own log, so an act is counted once whichever log holds it (ADR 0565 §11). +func (h *Healers) ActRecorded(body []byte) { + var a HandAct + if json.Unmarshal(body, &a) != nil || a.ID == "" { + return + } + h.HandActHeard(handacts.LogSeat+":"+a.ID, "put", body) +} + +// HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again. +func (h *Healers) HandActHeard(key, op string, value []byte) { + h.mu.Lock() + if h.acts == nil { + h.acts = map[string]HandAct{} + } + if op != "put" { + delete(h.acts, key) + } else { + var a HandAct + if json.Unmarshal(value, &a) == nil { + h.acts[key] = a + } + } + h.mu.Unlock() + h.Wanted() +} + +// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, +// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. +func (h *Healers) Wanted() []Observation { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + byCause := map[string][]HandAct{} + for _, a := range h.acts { + if personsDecision(a) || now.Sub(a.At) > Within { + continue + } + byCause[a.Cause] = append(byCause[a.Cause], a) + } + var causes []string + for c, acts := range byCause { + if len(acts) >= 2 { + causes = append(causes, c) + } + } + sort.Strings(causes) + var found []Observation + var next time.Time + for _, cause := range causes { + acts := byCause[cause] + sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) + var lines []string + for _, a := range acts { + lines = append(lines, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"), + strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why)) + } + newest := acts[len(acts)-1] + wanted := "a healer is wanted for it" + for _, r := range h.Registry { + if slices.Contains(r.Kinds, cause) { + wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its budget "+ + "is not enough", r.ID) + } + } + words := strings.NewReplacer(".", " ", "_", " ").Replace(cause) + found = append(found, Observation{Kind: "healer-wanted", Token: "healer-wanted", Severity: Warning, + Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause}, + Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts), + int(Within.Hours()/24), newest.By, wanted), + Said: strings.Join(lines, "; "), + Headline: capital(words) + " keeps being fixed by hand", + Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+ + "for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)), + Resolved: "Resolved: no more hand repairs of " + words}) + if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) { + next = leaves + } + } + h.see("S15", found) + if !next.IsZero() { + ttl := next.Sub(now).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil { + h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err) + } + } + return found +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go new file mode 100644 index 00000000..db056254 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -0,0 +1,423 @@ +//go:build integration + +package main + +// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's +// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An +// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the +// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the +// hour the brake stops every healer, said as an urgent finding on the bus. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + asked []string +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +// deliver hands the bundle one event as the runtime does from its consumer. +func (p *player) deliver(key string, body []byte) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("e-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{ + "key": key, "node": "ace", "body": json.RawMessage(body)}}}) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &ask) + p.mu.Lock() + p.asked = append(p.asked, ask.Key+" "+string(ask.Body)) + p.mu.Unlock() + if ask.Key != AssertTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + p.answer(m.ID, map[string]any{"says": "1 the bus lacks: created at this ask"}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +func raised(key, module string) []byte { + raw, _ := json.Marshal(map[string]any{"key": key, "kind": "consumer-lost", "severity": "warning", "resolver": "self", + "subject": map[string]any{"scope": "bus", "id": "EVENTS." + module}, "event": "condition-raised", + "show": health.ShowKeeper(key)}) + return raw +} + +// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes. +func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-healers does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState, + "mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + bucket := func(name string) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + t.Fatal(err) + } + return kv + } + keys := func(name string) []string { + k, _ := bucket(name).Keys(ctx) + return k + } + lease := bucket("mesh-controller_lease") + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil { + t.Fatal(err) + } + acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + healers := func() map[string]any { + raw, err := p.call(HealersTool, map[string]any{}) + if err != nil { + return nil + } + var a map[string]any + _ = json.Unmarshal(raw, &a) + return a + } + eventually(t, "the lease read as a controller's", func() bool { + a := healers() + return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller") + }) + + // The controller holds the lease: the induced failure is written as would, never acted on. + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 }) + if k := keys(Module + "_" + AttemptsState); len(k) != 0 { + t.Fatalf("acted under the controller's lease: %v", k) + } + // The write that hands the lease away: the same failure is acted on. + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "the healers acting", func() bool { return healers()["acting"] == true }) + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the act begun and finished on the bus", func() bool { + k := keys(Module + "_" + AttemptsState) + if len(k) != 1 { + return false + } + e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0]) + return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`) + }) + msg, err := acted.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) { + t.Fatalf("healer-acted was not said: %v", err) + } + p.mu.Lock() + asked := strings.Join(p.asked, "\n") + p.mu.Unlock() + if !strings.Contains(asked, AssertTool+` {"module":"records"}`) { + t.Errorf("the grants part was asked %s", asked) + } + // Eleven more acts within the hour: the twelfth was the limit, and the next condition meets the brake. + attempts := bucket(Module + "_" + AttemptsState) + for i := 0; i < BrakeLimit-1; i++ { + begun := time.Now().Add(-time.Duration(i+1) * time.Minute).UTC() + raw, _ := json.Marshal(Attempt{Healer: "H3", Condition: fmt.Sprintf("bus.EVENTS.m%d.missing", i), BudgetKey: "x", + Outcome: OutcomeActed, Begun: begun}) + if _, err := attempts.Put(ctx, attemptKey("H3", fmt.Sprintf("bus.EVENTS.m%d.missing", i), begun), raw); err != nil { + t.Fatal(err) + } + } + p.deliver(RaisedEvent, raised("bus.EVENTS.mail.missing", "mail")) + eventually(t, "the brake written as a finding on the bus", func() bool { + e, err := bucket(Module+"_"+FindingsState).Get(ctx, brakeKey) + if err != nil { + return false + } + f, err := health.DecodeFinding(e.Value()) + return err == nil && f.Severity == health.Urgent && f.Kind == "healers-braked" + }) + if n := len(keys(Module + "_" + AttemptsState)); n != BrakeLimit { + t.Errorf("past the brake the attempts are %d, want %d", n, BrakeLimit) + } +} diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod new file mode 100644 index 00000000..89b3df38 --- /dev/null +++ b/modules/mesh-healers/go.mod @@ -0,0 +1,22 @@ +module mesh-healers + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-healers/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json new file mode 100644 index 00000000..634dd388 --- /dev/null +++ b/modules/mesh-healers/module.json @@ -0,0 +1,82 @@ +{ + "module": "mesh-healers", + "version": "1", + "runs-as": "mesh-healers", + "claims": [ + { + "name": "mesh-healers", + "scope": "mesh", + "serves": [ + "healers", + "heal", + "drill" + ] + } + ], + "consumes": [ + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed", + "mesh-hand-acts.act-recorded" + ], + "invokes": [ + "mesh-grants.grants_assert", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-controller.drill" + ], + "reads": [ + "mesh-controller.lease", + "mesh-conditions.conditions", + "mesh-controller.hand-acts" + ], + "state": [ + { + "name": "attempts", + "key-ttl": true + }, + { + "name": "would", + "key-ttl": true + }, + "findings", + { + "name": "wakes", + "key-ttl": true + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-healers", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-healers", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-healers" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-healers", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-healers", + "binary": "mesh-healers", + "loads": [ + "mesh-healers" + ] + } + ] + } +} diff --git a/modules/mesh-issues/files/settings.json b/modules/mesh-issues/files/settings.json index 0231a55d..265fe2da 100644 --- a/modules/mesh-issues/files/settings.json +++ b/modules/mesh-issues/files/settings.json @@ -62,10 +62,13 @@ "mesh-catalog modules/mesh-build-node", "mesh-catalog modules/mesh-catalog", "mesh-catalog modules/mesh-cli", + "mesh-catalog modules/mesh-conditions", "mesh-catalog modules/mesh-config", "mesh-catalog modules/mesh-delivery", "mesh-catalog modules/mesh-delivery-gate", "mesh-catalog modules/mesh-files", + "mesh-catalog modules/mesh-hand-acts", + "mesh-catalog modules/mesh-healers", "mesh-catalog modules/mesh-issues", "mesh-catalog modules/mesh-knowledge", "mesh-catalog modules/mesh-mail", @@ -75,6 +78,7 @@ "mesh-catalog modules/mesh-provisioner", "mesh-catalog modules/mesh-provisioner-client", "mesh-catalog modules/mesh-reads", + "mesh-catalog modules/mesh-self-check", "mesh-catalog modules/mesh-vault", "mesh-catalog modules/mesh-watcher", "mesh-catalog modules/messenger", diff --git a/modules/mesh-issues/go.mod b/modules/mesh-issues/go.mod index 2aea2394..ea63f374 100644 --- a/modules/mesh-issues/go.mod +++ b/modules/mesh-issues/go.mod @@ -1,22 +1,22 @@ module mesh-issues -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nats.go v1.54.0 ) require ( github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/mesh-issues/go.sum b/modules/mesh-issues/go.sum index b16d5979..a2b8e018 100644 --- a/modules/mesh-issues/go.sum +++ b/modules/mesh-issues/go.sum @@ -1,27 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac h1:jnZ/n+fFMo7VQPNNXK1gjStHIJPw3SPbzPxsf/PpZqw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-issues/module.json b/modules/mesh-issues/module.json index d359bda2..196c586c 100644 --- a/modules/mesh-issues/module.json +++ b/modules/mesh-issues/module.json @@ -174,10 +174,13 @@ "mesh-catalog modules/mesh-build-node", "mesh-catalog modules/mesh-catalog", "mesh-catalog modules/mesh-cli", + "mesh-catalog modules/mesh-conditions", "mesh-catalog modules/mesh-config", "mesh-catalog modules/mesh-delivery", "mesh-catalog modules/mesh-delivery-gate", "mesh-catalog modules/mesh-files", + "mesh-catalog modules/mesh-hand-acts", + "mesh-catalog modules/mesh-healers", "mesh-catalog modules/mesh-issues", "mesh-catalog modules/mesh-knowledge", "mesh-catalog modules/mesh-mail", @@ -187,6 +190,7 @@ "mesh-catalog modules/mesh-provisioner", "mesh-catalog modules/mesh-provisioner-client", "mesh-catalog modules/mesh-reads", + "mesh-catalog modules/mesh-self-check", "mesh-catalog modules/mesh-vault", "mesh-catalog modules/mesh-watcher", "mesh-catalog modules/messenger", diff --git a/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go b/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go index 44f0a371..75412d92 100644 --- a/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go @@ -13,17 +13,19 @@ import ( _ "git.novox.be/novox/mesh-sdk/go/process" ) -// 1172/AC1: given mesh-mail's go.mod, when built, then it requires mesh-sdk go at -// v0.1.14-0.20261019062050-34392e74916c and go 1.26.0, and go/process is importable (the import above). +// 1172/AC1: given mesh-mail's go.mod, when built, then it requires mesh-sdk go at the release-2 commit or after it +// (v0.1.14-0.20261019062050-34392e74916c or any later pseudo-version, which the catalogue's pins move) and go 1.26.0, and +// go/process is importable (the import above). func TestGoModPinsTheSDKsProcessRelease(t *testing.T) { raw, err := os.ReadFile("../../go.mod") if err != nil { t.Fatal(err) } mod := string(raw) - const wantSDK = "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c" - if !strings.Contains(mod, wantSDK) { - t.Fatalf("go.mod does not require %q:\n%s", wantSDK, mod) + const prefix, release2 = "git.novox.be/novox/mesh-sdk/go v0.1.14-0.", "20261019062050" + at := strings.Index(mod, prefix) + if at < 0 || len(mod) < at+len(prefix)+14 || mod[at+len(prefix):at+len(prefix)+14] < release2 { + t.Fatalf("go.mod does not require mesh-sdk go at release 2 (%s) or after:\n%s", release2, mod) } if !strings.Contains(mod, "go 1.26.0") { t.Fatalf("go.mod does not declare go 1.26.0:\n%s", mod) diff --git a/modules/mesh-mail/go.mod b/modules/mesh-mail/go.mod index b5826dcc..6c9fdd03 100644 --- a/modules/mesh-mail/go.mod +++ b/modules/mesh-mail/go.mod @@ -3,7 +3,7 @@ module mesh-mail go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/emersion/go-imap/v2 v2.0.0-beta.8 github.com/emersion/go-message v0.18.2 github.com/jackc/pgx/v5 v5.11.0 diff --git a/modules/mesh-mail/go.sum b/modules/mesh-mail/go.sum index 8aa47987..d345f9ba 100644 --- a/modules/mesh-mail/go.sum +++ b/modules/mesh-mail/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c h1:/f6f2ugss172hDJSL3EFZqb9PdfO5DKzYVfPiX2LdAM= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/mesh-provisioner-client/README.md b/modules/mesh-provisioner-client/README.md index e4382024..7c62a365 100644 --- a/modules/mesh-provisioner-client/README.md +++ b/modules/mesh-provisioner-client/README.md @@ -4,19 +4,19 @@ Holds the `node-provisioner` seat on every node: the provisioner's client (novox P5). Go, one binary (`cmd/mesh-provisioner-client`), run as a service of its own under the account `mesh-provisioner-client`, outside the tool runner, on a bus account of its own, `provisioner.`, delivered as its own secret `provisioner-broker` (mesh-controller `internal/broker/provisioner.go`). The bus grants that account -its own machine's keys and, to read alone, the whole assignments mirror, where a provider's consumers on any -machine are bound to it (ADR 0373); nothing else. +its own machine's keys and, to read alone, the seat mesh-provisioner's whole `plans` and the controller's whole +assignments mirror, where a provider's consumers on any machine are bound to it (ADR 0373); nothing else. ## What it does -For each module the controller handed to the provisioner on its node — an assignment in -`mesh-controller_assignments` (`.`) whose value carries a `provisioning` plan: the templates archive +For each module placed on its node — a plan in the seat mesh-provisioner's `plans` (`mesh-provisioner_plans`, +`.`, novox/hq ADR 0570 clause 2), or, for a module the seat holds no plan of, until the controller hands +its lease away, an assignment in `mesh-controller_assignments` whose value carries a `provisioning` plan; a module +leaves only when neither places it: the templates archive by reference and digest, the files it fills (path, template, `owner-at-runtime`, how its program takes a change) and the placement values its templates name — the client: - fetches the templates archive and uses it only when it hashes to the plan's digest, keeping it by digest; -- records each placeholder as a need in `mesh-provisioner_needs`, `...` (a character a - key cannot hold written `=`); - watches each owner's keys for its node: placement in the plan, config in `mesh-config_resolved` (`..`), both named by `mesh-provisioner_registry`; - asks the vault (`mesh-vault.get`, named by the registry) for each of the module's own secrets its plan names @@ -58,7 +58,7 @@ and the placement values its templates name — the client: 4): every file of the module goes through the node-engine again, so an `owned` file changed on disk is put back; - reports `prov.applied..` and keeps the same report in `mesh-provisioner_applied` (`.`): `applied` with each file's action and digest, `waiting`, `failed` with the reason, or `removed`; -- when a module is taken off the node, removes its needs, its items and its report (the node-engine removes the files its +- when a module is taken off the node, removes its items and its report (the node-engine removes the files its declaration no longer names); and when the bus comes back, or the client starts, fills from the current values; - fills a provider's `${consumers:}` (novox/hq ADR 0373, issue 679) with one entry per consumer bound to it for that provision, in the order of the assignments' keys and, within a plan, of its `provisions` entries: a diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go index fc5dcfae..329d4d04 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go @@ -123,7 +123,7 @@ func (f fakeEntry) Operation() jetstream.KeyValueOp { return f.op } // afresh, whichever way it left or returned, so it is never filled with a generation held from before; a module // that stays placed keeps using what it holds. func TestAModulePlacedAgainAsksTheVaultAfreshAndAModuleThatStaysDoesNot(t *testing.T) { - const assignment = `{"node":"g14","module":"build","provisioning":{}}` + const assignment = `{"v":1,"node":"g14","module":"build","plan":{}}` // the seat's plan (ADR 0570 clause 2) own := VaultName("g14", "build", "bus-password") pair := PairName("g14", "db", "g14", "build", "") other := VaultName("g14", "gitea", "token") @@ -133,15 +133,15 @@ func TestAModulePlacedAgainAsksTheVaultAfreshAndAModuleThatStaysDoesNot(t *testi again int // asks of the vault after the first three: the module's own secret once more, or none }{ {name: "it stays placed: the assignment is put again", again: 0, steps: func(c *Client, s *state) { - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) }}, {name: "its assignment is deleted and put again", again: 1, steps: func(c *Client, s *state) { - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", op: jetstream.KeyValueDelete}}) - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", op: jetstream.KeyValueDelete}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) }}, {name: "it left and the client was away: only the later assignment arrives", again: 1, steps: func(c *Client, s *state) { delete(s.assigned, "build") - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) }}, {name: "it leaves the machine", again: 1, steps: func(c *Client, s *state) { c.leave(context.Background(), nil, s, "build") @@ -152,7 +152,7 @@ func TestAModulePlacedAgainAsksTheVaultAfreshAndAModuleThatStaysDoesNot(t *testi c := &Client{Node: "g14", Secrets: fake, Now: time.Now} s := &state{secrets: map[string]heldSecret{}, plans: map[string]Plan{}, refused: map[string]string{}, assigned: map[string]bool{"build": true}, binds: map[string]binding{}, providing: map[string]bool{}, - leftOut: map[string]string{}, needs: map[string][]byte{}, applied: map[string]bool{}, + leftOut: map[string]string{}, applied: map[string]bool{}, reported: map[string][]byte{}, items: map[string]map[string]itemValue{}, recorded: map[string][]byte{}, unplaced: map[string]bool{}} n := Need{Kind: "secret", Name: "bus-password"} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go index 9cf10e90..844f58a6 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go @@ -73,10 +73,11 @@ func (r *toolRunner) hear() chan string { return heard } -// placed writes an assignment of a module the controller has not handed to the provisioner: placed, no plan. +// placed writes the plan of a module placed here with nothing to provision: an empty plan (ADR 0570: a placed +// module has a plan key, empty or not). func (b *aBus) placed(t *testing.T, module string) { t.Helper() - b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z"}`, + b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"provisioning":{}}`, node, module)) } @@ -119,7 +120,7 @@ func TestNatsAModulesCodeReadsItsConfigAndIsToldOfAChange(t *testing.T) { } // Taken off the machine: said, and no longer answered. - b.del(t, AssignmentsBucket, node+".claude-code") + b.del(t, PlansBucket, node+".claude-code") until(t, heard, `claude-code {"kind":"config"}`) if a := runner.askConfig("claude-code", "managed_settings"); !strings.Contains(a.Error, "not placed on g14") { t.Fatalf("a module taken off the machine was answered %+v", a) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go index beacf539..e6815a3e 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go @@ -1,18 +1,21 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/provisions" + "bytes" "context" "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery" "os" "slices" "sort" "strings" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -39,11 +42,16 @@ import ( // The buckets the client reads and writes, and the subject it reports on. const ( - AssignmentsBucket = "mesh-controller_assignments" - ConfigBucket = "mesh-config_resolved" - RegistryBucket = "mesh-provisioner_registry" - NeedsBucket = "mesh-provisioner_needs" - AppliedBucket = "mesh-provisioner_applied" + // PlansBucket is where each module's provisioning plan on each node is read: the seat mesh-provisioner's + // `plans` (novox/hq ADR 0570 clause 2), read as an assignment's value carried it (asAssignment). + PlansBucket = provisions.PlansBucket + // ControllerAssignmentsBucket is the controller's assignments mirror, read for a module the seat holds no plan of + // until the controller hands its lease away: the holder writes plans only for modules with a want, so a module + // that runs without one (nats on novox) would otherwise lose its files' fill (review of 2026-10-30). + ControllerAssignmentsBucket = "mesh-controller_assignments" + ConfigBucket = "mesh-config_resolved" + RegistryBucket = "mesh-provisioner_registry" + AppliedBucket = "mesh-provisioner_applied" ) // AppliedSubject is where the client says what it applied for one module (ADR 0349 clauses 4 and 15). @@ -82,16 +90,6 @@ type FileApplied struct { Takes string `json:"takes,omitempty"` } -// NeedRecord is one need a module's templates name, as mesh-provisioner_needs keeps it at -// `...`: the key's name in a form the bus takes, and the name as written here. -type NeedRecord struct { - Node string `json:"node"` - Module string `json:"module"` - Kind string `json:"kind"` - Name string `json:"name"` - Files []string `json:"files"` -} - // Answer is a registry's entry, as mesh-provisioner keeps it: which domain answers a kind, and where. type Answer struct { Kind string `json:"kind"` @@ -134,12 +132,14 @@ type Client struct { type state struct { registry map[string]Answer plans map[string]Plan - refused map[string]string // a module whose plan cannot be acted on, and why - config map[string]map[string]json.RawMessage - needs map[string][]byte // key → value recorded - applied map[string]bool // module → a report is kept - reported map[string][]byte // module → the last report said, undated - secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere + // seatPlans and controllerAssigned are each source's value by key, as an assignment's value reads: the seat's + // plan wins, and the controller's assignment stands for a module the seat has no plan of. + seatPlans, controllerAssigned map[string][]byte + refused map[string]string // a module whose plan cannot be acted on, and why + config map[string]map[string]json.RawMessage + applied map[string]bool // module → a report is kept + reported map[string][]byte // module → the last report said, undated + secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere // items is every item recorded for this node, by collection and contributor: what a collector is filled from. items map[string]map[string]itemValue // recorded is each item's key and its value as kept: what this client compares before it records again. @@ -181,7 +181,7 @@ func (c *Client) Run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{AssignmentsBucket, ConfigBucket, RegistryBucket, NeedsBucket, AppliedBucket, + for _, b := range []string{PlansBucket, ControllerAssignmentsBucket, ConfigBucket, RegistryBucket, AppliedBucket, CollectBucket} { kv, err := c.JS.KeyValue(ctx, b) if err != nil { @@ -220,12 +220,12 @@ func (c *Client) Run(ctx context.Context) error { mine := c.Node + ".>" // The registry whole, and every machine's assignment: a provider here is bound by consumers on any machine, in // their plans (ADR 0373). Every other bucket, this machine's keys alone. - for _, b := range []string{RegistryBucket, AssignmentsBucket} { + for _, b := range []string{RegistryBucket, PlansBucket, ControllerAssignmentsBucket} { if err := watch(b, ">"); err != nil { return err } } - for _, b := range []string{ConfigBucket, NeedsBucket, AppliedBucket, CollectBucket} { + for _, b := range []string{ConfigBucket, AppliedBucket, CollectBucket} { if err := watch(b, mine); err != nil { return err } @@ -307,7 +307,7 @@ func (c *Client) Run(ctx context.Context) error { defer func() { _ = asks.Unsubscribe() }() s := &state{registry: map[string]Answer{}, plans: map[string]Plan{}, refused: map[string]string{}, - config: map[string]map[string]json.RawMessage{}, needs: map[string][]byte{}, applied: map[string]bool{}, + config: map[string]map[string]json.RawMessage{}, applied: map[string]bool{}, reported: map[string][]byte{}, secrets: map[string]heldSecret{}, items: map[string]map[string]itemValue{}, recorded: map[string][]byte{}, assigned: map[string]bool{}, binds: map[string]binding{}, providing: map[string]bool{}, leftOut: map[string]string{}, bound: map[string]map[string][]byte{}, @@ -315,7 +315,7 @@ func (c *Client) Run(ctx context.Context) error { // Every current value first: nothing is filled from half of them. current := map[string]bool{} var early []*nats.Msg - for len(current) < 6 { + for len(current) < 6 { // the registry, the plans, the controller's assignments, config, applied, collect select { case <-ctx.Done(): return ctx.Err() @@ -491,7 +491,7 @@ func (c *Client) told(u update) { if module, name, ok := strings.Cut(rest, "."); ok { c.tell(module, Changed{Kind: AskConfig, Key: name}) } - case AssignmentsBucket: + case PlansBucket, ControllerAssignmentsBucket: c.tell(rest, Changed{Kind: AskConfig}) } } @@ -539,16 +539,40 @@ func (c *Client) take(s *state, u update) []string { } // A kind's answer moves what every module waits on. return sortedKeys(s.plans) - case AssignmentsBucket: + case PlansBucket, ControllerAssignmentsBucket: + if s.seatPlans == nil { + s.seatPlans, s.controllerAssigned = map[string][]byte{}, map[string][]byte{} + } + source := s.seatPlans + if u.bucket == ControllerAssignmentsBucket { + source = s.controllerAssigned + } + if gone { + delete(source, key) + } else if u.bucket == ControllerAssignmentsBucket { + source[key] = e.Value() + } else if v, err := asAssignment(e.Value()); err != nil { + logf("[provisioner-client] the plan %s cannot be read, so the controller's assignment is read: %v", key, err) + delete(source, key) + } else { + source[key] = v + } + // The seat's plan where it holds one, the controller's assignment otherwise: a module leaves only when + // neither places it, never because the seat has no plan of it yet. + value, placed := s.seatPlans[key] + if !placed { + value, placed = s.controllerAssigned[key] + } + gone = !placed // A value that is not a standing assignment (ended, or in a state: ADR 0408 clause 4) is read as a removed // key, so the module leaves the machine as it does when the key goes (issue 1098). - gone = gone || !provider.Standing(e.Value()) + gone = gone || !provider.Standing(value) // Whom it binds, whichever machine's it is: a provider here listing its consumers is filled again. var providers []string if ended, had := s.binds[key]; gone && had { c.forgetBound(s, ended) } - if takeBinds(s, key, e.Value(), gone) { + if takeBinds(s, key, value, gone) { providers = sortedKeys(s.providing) } module, mine := strings.CutPrefix(key, c.Node+".") @@ -571,7 +595,7 @@ func (c *Client) take(s *state, u update) []string { logf("[provisioner-client] %s: its assignment changed, so the vault is asked again for %s", module, strings.Join(dropped, ", ")) } - m, p, err := planOf(c.Node, key, e.Value()) + m, p, err := planOf(c.Node, key, value) switch { case errors.Is(err, errNoPlan): case err != nil: @@ -597,12 +621,6 @@ func (c *Client) take(s *state, u update) []string { s.config[module][name] = r.Value } return []string{module} - case NeedsBucket: - if gone { - delete(s.needs, key) - } else { - s.needs[key] = e.Value() - } case AppliedBucket: module := strings.TrimPrefix(key, c.Node+".") s.applied[module] = !gone @@ -652,14 +670,9 @@ func (c *Client) take(s *state, u update) []string { return nil } -// leftModules are the modules with needs or a report kept and no assignment with a plan. +// leftModules are the modules with a report or items kept and no assignment with a plan. func (c *Client) leftModules(s *state) []string { left := map[string]bool{} - for key := range s.needs { - if parts := strings.SplitN(key, ".", 3); len(parts) == 3 && !hasPlan(s, parts[1]) && s.refused[parts[1]] == "" { - left[parts[1]] = true - } - } for m, kept := range s.applied { if kept && !hasPlan(s, m) && s.refused[m] == "" { left[m] = true @@ -673,9 +686,8 @@ func (c *Client) leftModules(s *state) []string { return sortedKeys(left) } -// leave removes a module's needs and its report: it is no longer the provisioner's on this node. +// leave removes a module's report and its items: it is no longer the provisioner's on this node. func (c *Client) leave(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string) { - prefix := c.Node + "." + module + "." delete(s.unplaced, module) // What it held of the vault goes with it: placed again, it asks afresh (issue 1945). c.forgetHeld(s, module) @@ -685,19 +697,8 @@ func (c *Client) leave(ctx context.Context, kvs map[string]jetstream.KeyValue, s delete(s.leftOut, k) } } - removed := 0 - for _, key := range sortedKeys(s.needs) { - if strings.HasPrefix(key, prefix) { - if err := kvs[NeedsBucket].Delete(ctx, key); err != nil { - logf("[provisioner-client] the need %s could not be removed: %v", key, err) - continue - } - delete(s.needs, key) - removed++ - } - } // Its items go: an item is in its collection only while its contributor is placed here. - removed += c.forgetItems(ctx, kvs, s, module, nil) + removed := c.forgetItems(ctx, kvs, s, module, nil) if !s.applied[module] && removed == 0 && s.reported[module] == nil { return } @@ -754,7 +755,7 @@ func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValu for _, it := range items { named(it.id(), it.text) } - c.recordNeeds(ctx, kvs, s, module, order, needs) + _ = needs // the client records no need (ADR 0570 clause 1): the seat composes them values := map[Need]string{} var failed []string @@ -930,52 +931,6 @@ func (c *Client) valueOf(ctx context.Context, s *state, module string, p Plan, n return "", "", fmt.Sprintf("%s: this client does not fill %s placeholders yet", n, n.Kind) } -// recordNeeds makes the module's needs in mesh-provisioner_needs what its templates name: each need written when it -// is new or names other files, each one no longer named removed. -func (c *Client) recordNeeds(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string, - order []Need, files map[Need][]string) { - want := map[string][]byte{} - for _, n := range order { - v, _ := json.Marshal(NeedRecord{Node: c.Node, Module: module, Kind: n.Kind, Name: n.Name, Files: files[n]}) - want[NeedKey(c.Node, module, n)] = v - } - for _, key := range sortedKeys(want) { - if bytes.Equal(s.needs[key], want[key]) { - continue - } - if _, err := kvs[NeedsBucket].Put(ctx, key, want[key]); err != nil { - logf("[provisioner-client] the need %s could not be recorded: %v", key, err) - continue - } - s.needs[key] = want[key] - } - prefix := c.Node + "." + module + "." - for _, key := range sortedKeys(s.needs) { - if _, still := want[key]; strings.HasPrefix(key, prefix) && !still { - if err := kvs[NeedsBucket].Delete(ctx, key); err != nil { - logf("[provisioner-client] the need %s could not be removed: %v", key, err) - continue - } - delete(s.needs, key) - } - } -} - -// NeedKey is a need's key in mesh-provisioner_needs, `...`: the name with every -// character a key cannot hold, and `=`, written `=`, so `${seat:mesh-broker:4222}` is `seat.mesh-broker=3A4222`. -func NeedKey(node, module string, n Need) string { - var b strings.Builder - for _, r := range []byte(n.Name) { - switch { - case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '/', r == '.': - b.WriteByte(r) - default: - fmt.Fprintf(&b, "=%02X", r) - } - } - return node + "." + module + "." + n.Kind + "." + b.String() -} - // report says a module's report on prov.applied and keeps it in mesh-provisioner_applied, when it says something // other than the last. func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, r Report) { @@ -1151,3 +1106,21 @@ func (c *Client) forgetItems(ctx context.Context, kvs map[string]jetstream.KeyVa } return removed } + +// asAssignment is a plan of the seat mesh-provisioner as the client's rules read an assignment's value: +// `{node, module, provisioning: plan}`. +func asAssignment(raw []byte) ([]byte, error) { + if _, err := provisions.DecodePlan(raw); err != nil { + return nil, err + } + // The plan as it is kept, its bytes unchanged: the client reads it strictly (planOf). + var kept struct { + Node string `json:"node"` + Module string `json:"module"` + Plan json.RawMessage `json:"plan"` + } + if err := json.Unmarshal(raw, &kept); err != nil { + return nil, err + } + return json.Marshal(map[string]any{"node": kept.Node, "module": kept.Module, "provisioning": kept.Plan}) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go index 1ad81773..fb0734ee 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go @@ -81,7 +81,7 @@ func startBus(t *testing.T) *aBus { t.Fatal(err) } b := &aBus{url: s.ClientURL(), domain: js, conn: nc, kv: map[string]jetstream.KeyValue{}} - for _, name := range []string{AssignmentsBucket, ConfigBucket, RegistryBucket, NeedsBucket, AppliedBucket, + for _, name := range []string{PlansBucket, ControllerAssignmentsBucket, ConfigBucket, RegistryBucket, AppliedBucket, CollectBucket, delivery.HaveBucket} { kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: name}) if err != nil { @@ -98,13 +98,25 @@ func startBus(t *testing.T) *aBus { } for _, kind := range placementKinds { b.put(t, RegistryBucket, kind, `{"kind":"`+kind+`","domain":"mesh-controller","from":"bucket",`+ - `"bucket":"mesh-controller_assignments","key":"."}`) + `"bucket":"mesh-provisioner_plans","key":"."}`) } return b } func (b *aBus) put(t *testing.T, bucket, key, value string) { t.Helper() + if bucket == PlansBucket { + // The seat mesh-provisioner keeps a plan, not an assignment (ADR 0570 clause 2): a module with no plan, or an + // ended one, has no key. + raw, ok := planned(value) + if !ok { + if err := b.kv[bucket].Delete(context.Background(), key); err != nil { + t.Fatal(err) + } + return + } + value = string(raw) + } if _, err := b.kv[bucket].PutString(context.Background(), key, value); err != nil { t.Fatalf("writing %s in %s: %v", key, bucket, err) } @@ -131,7 +143,7 @@ func (b *aBus) assign(t *testing.T, module string, plan Plan) { if err != nil { t.Fatal(err) } - b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ + b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ `"provisioning":%s}`, node, module, p)) } @@ -430,11 +442,6 @@ func TestNatsATemplateFillsFromEachOwnersKeys(t *testing.T) { if r.Templates == "" || len(r.Files) != 1 || r.Files[0].Action != "created" || r.Files[0].Takes != "reload-on" { t.Errorf("prov.applied said %+v", r) } - // The needs, one per placeholder, under the node's and the module's name. - eventually(t, "the needs are recorded", func() bool { - got := b.keys(t, NeedsBucket, node+".dunst.") - return len(got) == 2 && got[0] == "g14.dunst.config.font-size" && got[1] == "g14.dunst.machine.account-home" - }) kept, err := b.kv[AppliedBucket].Get(context.Background(), node+".dunst") if err != nil || !strings.Contains(string(kept.Value()), `"status":"applied"`) { t.Errorf("mesh-provisioner_applied keeps %v (%v)", kept, err) @@ -603,17 +610,16 @@ func TestNatsWithNothingOverTheBusItAsksForNoWriteNoSecretAndNoReport(t *testing } } -func TestNatsAModuleTakenOffTheNodeLosesItsNeedsAndItsReport(t *testing.T) { +func TestNatsAModuleTakenOffTheNodeLosesItsReport(t *testing.T) { b := startBus(t) heard := b.listen(t) e := startEngine(t, map[string]string{dunstrc: "owned"}) b.config(t, "dunst", "font-size", "13") b.assign(t, "dunst", dunst(t)) aClient(t, b, e, t.TempDir()) - eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) + eventually(t, "its report is kept", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 1 }) - b.del(t, AssignmentsBucket, node+".dunst") - eventually(t, "its needs go", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 0 }) + b.del(t, PlansBucket, node+".dunst") eventually(t, "its report goes", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 0 }) eventually(t, "prov.applied says it is removed", func() bool { r, _ := heard.last("dunst") @@ -621,6 +627,44 @@ func TestNatsAModuleTakenOffTheNodeLosesItsNeedsAndItsReport(t *testing.T) { }) } +// Review of 2026-10-30: until the controller hands its lease away, a module the seat holds no plan of is read from +// the controller's assignment (nats on novox runs without a want, so the holder writes it no plan), and a module never +// leaves because its plan is missing: only when neither source places it. +func TestNatsAModuleWithoutASeatPlanIsFilledFromTheControllersAssignmentAndLeavesOnlyWhenNeitherPlacesIt(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + p, err := json.Marshal(dunst(t)) + if err != nil { + t.Fatal(err) + } + assignment := fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z","provisioning":%s}`, node, p) + b.put(t, ControllerAssignmentsBucket, node+".dunst", assignment) + aClient(t, b, e, t.TempDir()) + eventually(t, "dunstrc is filled from the controller's assignment", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 13") + }) + + // The seat's plan comes, and goes again: the module stays, filled, its report kept. + b.assign(t, "dunst", dunst(t)) + b.del(t, PlansBucket, node+".dunst") + b.config(t, "dunst", "font-size", "15") + eventually(t, "dunstrc follows a change with only the controller's assignment", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 15") + }) + if r, _ := heard.last("dunst"); r.Status == StatusRemoved || len(b.keys(t, AppliedBucket, node+".dunst")) != 1 { + t.Fatalf("the module left when its plan went, though the controller still places it: %+v", r) + } + + // Neither places it: it leaves. + b.del(t, ControllerAssignmentsBucket, node+".dunst") + eventually(t, "prov.applied says it is removed", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusRemoved + }) +} + func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) { b := startBus(t) const zshrc = "/home/nox/.zshrc" @@ -640,14 +684,13 @@ func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) { // Away: a value changes and a module leaves, and the artifact store is gone. stop() b.config(t, "dunst", "font-size", "17") - b.del(t, AssignmentsBucket, node+".zsh") + b.del(t, PlansBucket, node+".zsh") heard := b.listen(t) aClient(t, b, e, state) eventually(t, "dunstrc has the value set while it was away", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 17") }) - eventually(t, "the module that left lost its needs", func() bool { return len(b.keys(t, NeedsBucket, node+".zsh.")) == 0 }) eventually(t, "and its report", func() bool { r, _ := heard.last("zsh") return r.Status == StatusRemoved && len(b.keys(t, AppliedBucket, node+".zsh")) == 0 @@ -662,7 +705,7 @@ func TestNatsWhatCannotBeFilledFailsWithTheReason(t *testing.T) { e := startEngine(t, map[string]string{dunstrc: "owned", "/etc/app.conf": "owned"}) aClient(t, b, e, t.TempDir()) - b.put(t, AssignmentsBucket, node+".broken", `{"node":"g14","module":"broken","provisioning":{"templates":`+ + b.put(t, PlansBucket, node+".broken", `{"node":"g14","module":"broken","provisioning":{"templates":`+ `{"source":"x","digest":"sha256:00"},"files":[],"surprise":1}}`) p := dunst(t) p.Templates.Digest = "sha256:" + strings.Repeat("0", 64) @@ -683,19 +726,6 @@ func TestNatsWhatCannotBeFilledFailsWithTheReason(t *testing.T) { } } -func TestANeedsKeyHoldsAnyName(t *testing.T) { - for n, want := range map[Need]string{ - {Kind: "config", Name: "font-size"}: "g14.dunst.config.font-size", - {Kind: "seat", Name: "mesh-broker:4222"}: "g14.dunst.seat.mesh-broker=3A4222", - {Kind: "config", Name: "a=b c"}: "g14.dunst.config.a=3Db=20c", - {Kind: "machine", Name: "account-home"}: "g14.dunst.machine.account-home", - } { - if got := NeedKey(node, "dunst", n); got != want { - t.Errorf("%v: %s, want %s", n, got, want) - } - } -} - // TestNatsResticsFilesFillFromTheBackupPlacement is the backup lines as placement (issue 794): restic's two files // are the plan's `backup:lines` and `backup:data`, each exactly the text the controller composed, trailing newline // and all, and a changed plan refills them. @@ -778,3 +808,21 @@ func TestNatsTheNodesOwnHaveMovingFillsEveryModuleAgain(t *testing.T) { t.Errorf("dunstrc was asked %d times for one change", after-before) } } + +// planned is an assignment's value as the seat mesh-provisioner keeps its plan, `{v, node, module, plan}`; false for one +// with no plan, or ended, or in a state, which has no key there. +func planned(assignment string) ([]byte, bool) { + var a struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning json.RawMessage `json:"provisioning"` + Ended json.RawMessage `json:"ended"` + State json.RawMessage `json:"state"` + } + if json.Unmarshal([]byte(assignment), &a) != nil || len(a.Provisioning) == 0 || string(a.Provisioning) == "null" || + len(a.Ended) > 0 || len(a.State) > 0 { + return nil, false + } + raw, _ := json.Marshal(map[string]any{"v": 1, "node": a.Node, "module": a.Module, "plan": a.Provisioning}) + return raw, true +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go index 71c7503a..defb001a 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go @@ -63,18 +63,13 @@ func TestNatsZshsFileHoldsEveryPlacedContributorsItemAndLosesOneWhoseContributor r, _ := heard.last("powerlevel10k") return r.Status == StatusApplied && len(r.Items) == 1 && r.Items[0] == "shell:zsh:normal" }) - // zsh records the collections it reads as its needs, like any value. - eventually(t, "zsh's needs are recorded", func() bool { - got := b.keys(t, NeedsBucket, node+".zsh.") - return len(got) == 2 && got[0] == "g14.zsh.collected.zsh-widgets" && got[1] == "g14.zsh.shell.zsh=3Anormal" - }) // A contributor's own value changes: its item, and so zshrc, follow. b.config(t, "fzf", "binding", `"^F"`) eventually(t, "zshrc follows the item", func() bool { return strings.Contains(e.read(t, zshrc), "bindkey ^F") }) // powerlevel10k leaves the node: its item goes, and zshrc without it. - b.del(t, AssignmentsBucket, node+".powerlevel10k") + b.del(t, PlansBucket, node+".powerlevel10k") without := "# zsh\n# mesh-cli\nalias nox=mesh-cli\n# then\nbindkey ^F fzf-file-widget\n# end\n" eventually(t, "zshrc loses the item of the contributor that left", func() bool { return e.read(t, zshrc) == without }) if got := b.keys(t, CollectBucket, node+".shell=3Azsh=3Anormal.powerlevel10k"); len(got) != 0 { @@ -98,7 +93,7 @@ func TestNatsACollectorStartingWithItsNodeHoldsTheItemsAlreadyThereAndOnesRecord eventually(t, "zshrc holds mesh-cli's item", func() bool { return strings.Contains(e.read(t, zshrc), "alias nox") }) stop() - b.del(t, AssignmentsBucket, node+".mesh-cli") + b.del(t, PlansBucket, node+".mesh-cli") b.assign(t, "zsh-autosuggestions", contributor(t, "shell:zsh:normal", "source suggestions.zsh\n")) aClient(t, b, e, state) want := "# zsh\n# zsh-autosuggestions\nsource suggestions.zsh\n# then\n# end\n" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go index e48d403b..048e218b 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go @@ -35,7 +35,7 @@ func (b *aBus) bind(t *testing.T, at, module string, provisions ...PlannedProvis if err != nil { t.Fatal(err) } - b.put(t, AssignmentsBucket, at+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ + b.put(t, PlansBucket, at+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ `"provisioning":{"templates":{"source":"x","digest":"sha256:%s"},"files":[{"id":"c","path":"/etc/c",`+ `"template":"c","owner-at-runtime":"owned"}],"provisions":%s}}`, at, module, strings.Repeat("0", 64), ps)) } @@ -140,7 +140,7 @@ func TestNatsAProvidersConsumersAreFilledFromTheVaultAndFollowEachBindAndRemake( return sameList(consumersIn(t, e), letta, photos, gitea) }) // A consumer unbinds. - b.del(t, AssignmentsBucket, "shanks.gitea") + b.del(t, PlansBucket, "shanks.gitea") eventually(t, "a consumer that unbinds leaves the file", func() bool { return sameList(consumersIn(t, e), letta, photos) }) @@ -160,11 +160,6 @@ func TestNatsAProvidersConsumersAreFilledFromTheVaultAndFollowEachBindAndRemake( t.Errorf("a consumer's credential crossed the bus in the clear: %q", said) } } - // The need is recorded as any other, by name: never a value. - if keys := b.keys(t, NeedsBucket, node+".postgres."); len(keys) != 1 || - keys[0] != node+".postgres.consumers.postgres-database" { - t.Errorf("the provider's needs are %v", keys) - } } // A provider with no consumer bound is filled with an empty list, never left waiting. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go deleted file mode 100644 index 996fdab9..00000000 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "testing" - "time" -) - -// Issue 1098: a value of the assignments bucket with `ended` set takes the module off the node exactly as a removed -// key does, and a value with the fields the controller adds to an assignment (build, commit, rollout, unknown ones) -// is read under the same key. -func TestNatsAnEndedAssignmentTakesAModuleOffTheNodeLikeARemovedKey(t *testing.T) { - b := startBus(t) - heard := b.listen(t) - e := startEngine(t, map[string]string{dunstrc: "owned"}) - b.config(t, "dunst", "font-size", "13") - p := dunst(t) - b.assign(t, "dunst", p) - aClient(t, b, e, t.TempDir()) - eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) - - b.put(t, AssignmentsBucket, node+".dunst", fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z",`+ - `"build":"b-1","commit":"abc","rollout":"r-1","ended":"2026-10-18T01:00:00Z",`+ - `"provisioning":%s}`, node, planJSON(t, p))) - eventually(t, "its needs go", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 0 }) - eventually(t, "its report goes", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 0 }) - eventually(t, "prov.applied says it is removed", func() bool { - r, _ := heard.last("dunst") - return r.Status == StatusRemoved - }) -} - -func TestNatsAnAssignmentWithTheNewFieldsIsFilledUnderTheSameKey(t *testing.T) { - b := startBus(t) - e := startEngine(t, map[string]string{dunstrc: "owned"}) - b.config(t, "dunst", "font-size", "13") - p := dunst(t) - b.assign(t, "dunst", p) - aClient(t, b, e, t.TempDir()) - eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) - b.put(t, AssignmentsBucket, node+".dunst", fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z",`+ - `"build":"b-1","artifact":{"name":"dunst","version":"1"},"commit":"abc","rollout":"r-1",`+ - `"since":"2026-10-18T00:00:00Z","ended":"","state":"","future":1,"provisioning":%s}`, node, planJSON(t, p))) - time.Sleep(300 * time.Millisecond) - if got := b.keys(t, NeedsBucket, node+".dunst."); len(got) != 2 { - t.Errorf("a value with the new fields took the module's needs away: %v", got) - } -} - -func planJSON(t *testing.T, p Plan) []byte { - t.Helper() - raw, err := json.Marshal(p) - if err != nil { - t.Fatal(err) - } - return raw -} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go index 8caf8968..43cc4dfd 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go @@ -13,16 +13,15 @@ import ( "strings" ) -// The provisioner's plan of a module on this node (novox/hq ADR 0349 clause 4, design 57 P5): placement's owner, -// the controller, carries it in the module's assignment, `.` of mesh-controller_assignments -// (mesh-controller internal/inventory Provisioning), which this client reads for its own machine alone. An -// assignment with no plan is a module the controller has not handed to the provisioner: its files are the -// node-engine's, from the declaration, and this client leaves them alone (ADR 0349 clause 17). A module only pulled -// in by another on this node, and handed, has a value of its own there, which names what requires it and no time it -// was assigned (issue 765): read and filled the same way. +// The provisioner's plan of a module on this node (novox/hq ADR 0349 clause 4, ADR 0570 clause 2): the seat +// mesh-provisioner composes it from bus state and keeps it in `plans`, `.`, which this client reads for +// its own machine (and every machine's, for a provider's consumers). asAssignment carries a plan into the Assignment +// it acts on. A module the seat holds no plan of is read from the controller's assignment until the controller hands its +// lease away; a module neither places is not placed here, and its files are the node-engine's, from the declaration. A module only pulled in by another on this +// node has a plan of its own there: read and filled the same way. -// Assignment is one value of the assignments mirror. **Lenient at the top**, so a field the controller adds to an -// assignment (design 54 §1) does not stop the client; **strict in the plan**, which is the client's to act on. +// Assignment is one plan as the client acts on it. **Lenient at the top**; **strict in the plan**, which is the +// client's to act on. type Assignment struct { Node string `json:"node"` Module string `json:"module"` diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go index 23388bec..a8b32af5 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go @@ -18,7 +18,7 @@ func (b *aBus) pullIn(t *testing.T, module, requiredBy string, plan Plan) { if err != nil { t.Fatal(err) } - b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"required-by":%q,`+ + b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"required-by":%q,`+ `"provisioning":%s}`, node, module, requiredBy, p)) } @@ -70,7 +70,7 @@ func TestNatsAPulledInModuleIsFilledWithItsSecretAndItsItemIsRecorded(t *testing return r.Status == StatusApplied && len(r.Files) == 2 }) // What required it lets it go: its item leaves the collection. - b.del(t, AssignmentsBucket, node+".mesh-cli") + b.del(t, PlansBucket, node+".mesh-cli") eventually(t, "zshrc loses the item of a module no longer pulled in", func() bool { return e.read(t, zshrc) == "# zsh\n# then\n# end\n" }) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go index 08ca9042..53831d21 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go @@ -2,12 +2,14 @@ package main import ( "context" + "encoding/json" "os" "strings" "sync" "testing" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" "github.com/nats-io/nats.go/jetstream" ) @@ -49,12 +51,32 @@ func (m *memBus) Delete(_ context.Context, bucket, key string) error { return nil } -// routeEntryFor is the entry route-proxy's helper writes for module's plan entry on the route provision. +// routeEntryFor is the entry route-proxy's helper writes for module's plan entry on the route provision, the plan +// as the seat mesh-provisioner keeps it (ADR 0570 clause 2). func routeEntryFor(t *testing.T, module, planEntry string) string { t.Helper() - bus := &memBus{buckets: map[string]map[string][]byte{provider.AssignmentsBucket: { - node + "." + module: []byte(`{"node":"` + node + `","module":"` + module + `","provisioning":{"provisions":[` + - planEntry + `]}}`)}}} + bus := &memBus{buckets: map[string]map[string][]byte{provisions.PlansBucket: { + node + "." + module: []byte(`{"v":1,"node":"` + node + `","module":"` + module + `","plan":{"provisions":[` + + planEntry + `]}}`)}, provisions.ProvisionsBucket: {}}} + // The key the seat composed for that entry, provided-for: asked of route-proxy (ADR 0570 clause 3). + var b struct { + Provision string `json:"provision"` + Local string `json:"local"` + } + if err := json.Unmarshal([]byte(planEntry), &b); err != nil { + t.Fatal(err) + } + pr := provisions.Provision{Provision: b.Provision, Local: b.Local, State: provisions.Asked, + Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "route-proxy"}} + key, err := pr.Key() + if err != nil { + t.Fatal(err) + } + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + bus.buckets[provisions.ProvisionsBucket][key] = raw p := provider.New(provider.Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: t.Logf, Offers: map[string]provider.Offer{"route": {Credential: provider.NoCredential, Serves: map[string]any{}}}}) out, err := p.Pass(context.Background()) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go index ff2a5e0b..c8c4149f 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go @@ -161,7 +161,9 @@ type changedSecret struct { } // secretOf is a module's secret, from what the run holds or from the vault: its value, or what it waits for, or -// why it cannot be filled. +// why it cannot be filled. The client fills it in this release: the node-engine of today writes a file's text as it +// is handed, and the new one fills only a provision key's reference, never a module's own secret (review of +// 2026-10-30); its fill is an addition, for provision keys, until it fills every secret. func (c *Client) secretOf(ctx context.Context, s *state, module string, p Plan, n Need) (value, wait, fail string) { name := "" if slices.ContainsFunc(p.Secrets, func(sec PlannedSecret) bool { return sec.Name == n.Name }) { diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go index 1538edb5..5c48fb45 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go @@ -170,10 +170,6 @@ func TestNatsAMovedModulesFileIsFilledWithItsSecretFromTheVault(t *testing.T) { r, _ := heard.last("app") return r.Status == StatusApplied && len(r.Files) == 2 }) - eventually(t, "the needs name the secrets, never their values", func() bool { - got := b.keys(t, NeedsBucket, node+".app.") - return strings.Join(got, " ") == "g14.app.config.font-size g14.app.secret.app-token g14.app.secret.db-password" - }) if wire.heard() == 0 { t.Fatal("the sniffer heard nothing: it proves nothing") } @@ -199,7 +195,7 @@ func TestNatsASecretRotatedInTheVaultReachesItsFileWithoutANewDeclaration(t *tes b.assign(t, "app", app(t)) aClient(t, b, e, t.TempDir()) eventually(t, "app.conf is filled", func() bool { return strings.Contains(e.read(t, appConf), "token = first\n") }) - assigned, err := b.kv[AssignmentsBucket].Get(context.Background(), node+".app") + assigned, err := b.kv[PlansBucket].Get(context.Background(), node+".app") if err != nil { t.Fatal(err) } @@ -212,7 +208,7 @@ func TestNatsASecretRotatedInTheVaultReachesItsFileWithoutANewDeclaration(t *tes r, _ := heard.last("app") return r.Status == StatusApplied && r.Files[0].Action == "updated" }) - now, err := b.kv[AssignmentsBucket].Get(context.Background(), node+".app") + now, err := b.kv[PlansBucket].Get(context.Background(), node+".app") if err != nil || now.Revision() != assigned.Revision() { t.Errorf("the module's assignment was written again: revision %d, was %d", now.Revision(), assigned.Revision()) } diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf index 23fcc2cb..783bf836 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf @@ -11,7 +11,7 @@ accounts { users = [ { user: "domain", password: "x" } { user: "provisioner.g14", password: "x", permissions: { - publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_needs.*.$KV.mesh-provisioner_needs.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_needs.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_needs.$KV.mesh-provisioner_needs.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-controller_assignments", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_needs", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-controller_assignments.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_needs.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "$KV.mesh-provisioner_needs.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } + publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.>", "$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-provisioner_plans", "$JS.API.STREAM.INFO.KV_mesh-controller_assignments", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-provisioner_plans.>", "$JS.FC.KV_mesh-controller_assignments.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } subscribe: { allow: ["_INBOX.provisioner.g14.>", "mesh.mod.mesh-vault.event.secret-changed", "prov.ask.g14"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/modules/mesh-provisioner-client/go.mod b/modules/mesh-provisioner-client/go.mod index a0fda0b8..cff5e3a4 100644 --- a/modules/mesh-provisioner-client/go.mod +++ b/modules/mesh-provisioner-client/go.mod @@ -3,7 +3,7 @@ module mesh-provisioner-client go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/sys v0.48.0 diff --git a/modules/mesh-provisioner-client/go.sum b/modules/mesh-provisioner-client/go.sum index c03eccc2..a2b8e018 100644 --- a/modules/mesh-provisioner-client/go.sum +++ b/modules/mesh-provisioner-client/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-provisioner/.gitignore b/modules/mesh-provisioner/.gitignore new file mode 100644 index 00000000..b51e2c5c --- /dev/null +++ b/modules/mesh-provisioner/.gitignore @@ -0,0 +1,2 @@ +/mesh-provisioner +/cmd/mesh-provisioner/mesh-provisioner diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go new file mode 100644 index 00000000..c8a8b357 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -0,0 +1,908 @@ +package main + +// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed +// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them +// in its states `plans` (`.`) and `provisions` (`..[.]`), which it alone +// writes. +// +// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the +// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants +// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a +// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5). +// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that +// consumer, `_bound` under the same key: present is provided (the vault's generation carried into the +// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by +// itself: a key of a provider that writes no outcome stays asked. +// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's +// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it; +// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/compose/fromruntime" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The states it keeps, beside Registry, Needs, Applied and Collect. +const ( + ProvisionsState = provisions.ProvisionsState + PlansState = provisions.PlansState +) + +// The reads it composes from. +const ( + PlacementRead = delivery.PlacementRead + WantsRead = delivery.AssignerModule + ".wants" + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" + BuildsRead = delivery.BuilderModule + ".builds" + HaveRead = "node-engine.have" + ResolvedRead = "mesh-config.resolved" +) + +// BoundReads are the providers' entries this module's manifest reads, `.bound`, whose changes it hears. Which +// path judges a provider is its manifest's word (keepsEntries), not this list: a provider that declares `bound` and is +// not read here leaves its keys asked, saying its entries are not read. +var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"} + +// boundRead is a provider's entries as this module reads them. +func boundRead(module string) string { return module + ".bound" } + +// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names +// (module → fingerprint), the newest build of every other placed module. +type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) + +// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's +// record of it. +type Asker interface { + Publish(subject string, body any, id string) error + Record(id string) ([]byte, bool, error) +} + +// Holder is the seat's holder. +type Holder struct { + Store Store + Compose Composer + Asks Asker + Now func() time.Time + Logf func(format string, args ...any) + + mu sync.Mutex +} + +// Composed is what one node's composition did. +type Composed struct { + Node string `json:"node"` + Plans []string `json:"plans-written,omitempty"` + Gone []string `json:"plans-removed,omitempty"` + Changed []string `json:"provisions-changed,omitempty"` + Asked []string `json:"retiring-asked,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +// ComposeNode composes one node: its plans and its provisions. +func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) { + h.mu.Lock() + defer h.mu.Unlock() + out := Composed{Node: node} + wanted, err := h.wantedOn(node) + if err != nil { + return out, err + } + // A module held on the node with no want (the bus on the control-node) is composed at the build its have says it + // runs, so it has a plan too and its client keeps filling it (review of 2026-10-30). + for module, fp := range h.runningOn(node) { + if _, ok := wanted[module]; !ok { + wanted[module] = fp + } + } + atWants, err := h.Compose(ctx, node, wanted) + if err != nil { + return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) + } + newest, err := h.Compose(ctx, node, nil) + if err != nil { + return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err) + } + if err := h.writePlans(node, atWants, &out); err != nil { + out.Problems = append(out.Problems, err.Error()) + } + needed := map[string]provisions.Provision{} + elsewhere := map[string]provisions.Provision{} + vaultNode := h.vaultNode() + for _, plans := range []map[string]catalogue.Plan{atWants, newest} { + for module, plan := range plans { + for _, p := range plan.Provisions { + consumer := provisions.End{Module: module, Node: node} + provider := provisions.End{Module: p.ProviderModule, Node: p.Provider} + pv := provisions.Provision{Provision: p.Provision, Local: p.Local, Consumer: consumer, Provider: provider, + DependsOn: provisions.DependsOn(consumer, p, h.providerPlan(p, node, atWants))} + key, err := pv.Key() + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if _, seen := needed[key]; !seen { + needed[key] = pv + } + // Its credential is a provision of its own, the vault's (clause 6): on the consumer's node for a value of the + // consumer's own, on the provider's for the provider's one shared value. + ck, ok := provisions.CredentialKey(p.CredentialForm, consumer, provider, p.Provision, p.Local) + if !ok { + continue + } + cn, cm, cp, cl, err := provisions.ParseKey(ck) + if err != nil { + continue + } + if vaultNode == "" { + out.Problems = append(out.Problems, "no node runs mesh-vault, so the credential "+ck+" has no provider") + continue + } + cred := provisions.Provision{Provision: cp, Local: cl, Consumer: provisions.End{Module: cm, Node: cn}, + Provider: provisions.End{Module: provisions.VaultModule, Node: vaultNode}} + if cn == node { + if _, seen := needed[ck]; !seen { + needed[ck] = cred + } + } else { + elsewhere[ck] = cred + } + } + } + } + h.judgeElsewhere(elsewhere, &out) + h.judgeNode(node, needed, &out) + return out, nil +} + +// providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else +// as kept in `plans`; nil when the provider is not handed. +func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan { + if p.Provider == node { + if plan, ok := here[p.ProviderModule]; ok { + return &plan + } + return nil + } + key, err := provisions.PlanKey(p.Provider, p.ProviderModule) + if err != nil { + return nil + } + e, err := h.Store.Get(PlansState, key) + if err != nil || e == nil { + return nil + } + plan, err := provisions.DecodePlan(e.Value) + if err != nil { + return nil + } + return &plan.Plan +} + +// vaultNode is the node mesh-vault is placed on: the provider of every credential. +func (h *Holder) vaultNode() string { + nodes := h.placedOn(provisions.VaultModule) + if len(nodes) == 0 { + return "" + } + return nodes[0] +} + +// dependedOn is every key of a node that a provision of another node depends on. +func (h *Holder) dependedOn(node string) []string { + var out []string + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return nil + } + for _, k := range keys { + if strings.HasPrefix(k, node+".") { + continue + } + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + if p, err := provisions.Decode(e.Value); err == nil { + for _, d := range p.DependsOn { + if strings.HasPrefix(d, node+".") { + out = append(out, d) + } + } + } + } + return out +} + +// judgeElsewhere writes each needed key of another node (a provider's shared credential), its state from its outcome; +// that node's own composition keeps it while a provision depends on it. +func (h *Holder) judgeElsewhere(keys map[string]provisions.Provision, out *Composed) { + for _, key := range sortedKeys(keys) { + next := keys[key] + var have provisions.Provision + e, err := h.Store.Get(ProvisionsState, key) + if err == nil && e != nil { + if have, err = provisions.Decode(e.Value); err == nil { + next.Reference = have.Reference + } + } + h.outcome(key, &next) + if e != nil && same(have, next) { + continue + } + h.put(key, next) + out.Changed = append(out.Changed, key+" "+next.State) + } +} + +// wantedOn is the fingerprint each want on a node names, by module. +func (h *Holder) wantedOn(node string) (map[string]string, error) { + out := map[string]string{} + keys, err := h.Store.Keys(WantsRead) + if err != nil { + return nil, fmt.Errorf("the wants cannot be read: %w", err) + } + for _, k := range keys { + n, module, err := delivery.ParseWantKey(k) + if err != nil || n != node { + continue + } + e, err := h.Store.Get(WantsRead, k) + if err != nil || e == nil { + continue + } + if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" { + out[module] = w.FP + } + } + return out, nil +} + +// runningOn is the fingerprint each module's have on a node says it runs, by module: what a module held without a want +// is composed at. +func (h *Holder) runningOn(node string) map[string]string { + out := map[string]string{} + keys, err := h.Store.Keys(HaveRead) + if err != nil { + return out + } + for _, k := range keys { + n, module, isNode, err := delivery.ParseHaveKey(k) + if err != nil || isNode || n != node { + continue + } + e, err := h.Store.Get(HaveRead, k) + if err != nil || e == nil { + continue + } + if have, err := delivery.DecodeHave(e.Value); err == nil && have.FP != "" { + out[module] = have.FP + } + } + return out +} + +// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. +func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { + var problems []error + for _, module := range sortedKeys(plans) { + key, err := provisions.PlanKey(node, module) + if err != nil { + problems = append(problems, err) + continue + } + raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]}) + if err != nil { + problems = append(problems, err) + continue + } + if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) { + continue + } + if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil { + problems = append(problems, fmt.Errorf("the plan %s: %w", key, err)) + continue + } + out.Plans = append(out.Plans, key) + } + keys, err := h.Store.Keys(PlansState) + if err != nil { + return errors.Join(append(problems, err)...) + } + for _, k := range keys { + n, module, ok := strings.Cut(k, ".") + if !ok || n != node { + continue + } + if _, still := plans[module]; still { + continue + } + if err := h.Store.Delete(PlansState, k); err != nil { + problems = append(problems, err) + continue + } + out.Gone = append(out.Gone, k) + } + return errors.Join(problems...) +} + +// judgeNode brings the node's provision keys to what is needed and what each provider said. +func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) { + held := h.held(node) + // A key of this node another node's provision depends on is needed (a provider's one shared credential). + for _, dep := range h.dependedOn(node) { + if have, ok := held[dep]; ok { + if _, n := needed[dep]; !n { + have.State, have.Why, have.Ask, have.Warrant = "", "", "", nil + needed[dep] = have + } + } + } + for _, key := range unionKeys(needed, held) { + want, isNeeded := needed[key] + have, isHeld := held[key] + var next provisions.Provision + switch { + case isNeeded: + next = want + if isHeld { + next.Reference = have.Reference + } + h.outcome(key, &next) + case have.State == provisions.Retiring: + next = have + if next.Ask == "" && next.Warrant == nil { + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if gone, why := h.retired(key, have); gone { + if err := h.Store.Delete(ProvisionsState, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Changed = append(out.Changed, key+" (retired: "+why+")") + } + continue + } + default: + next = have + next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+ + "any more", node, have.Provision, have.Provider.Module, have.Provider.Node) + next.Warrant = nil + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if isHeld && same(have, next) { + continue + } + next.At = h.Now().UTC() + raw, err := provisions.Encode(next) + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err)) + continue + } + out.Changed = append(out.Changed, key+" "+next.State) + } +} + +// held is the node's provision keys as kept. +func (h *Holder) held(node string) map[string]provisions.Provision { + out := map[string]provisions.Provision{} + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return out + } + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + if p, err := provisions.Decode(e.Value); err == nil { + out[k] = p + } + } + return out +} + +// same says two provisions say the same, their time aside. +func same(a, b provisions.Provision) bool { + a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0 + ra, _ := json.Marshal(a) + rb, _ := json.Marshal(b) + return bytes.Equal(ra, rb) +} + +// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what +// the holder needs and nothing more. +type entry struct { + // State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits. + State string `json:"state,omitempty"` + Why string `json:"why,omitempty"` + Refused string `json:"refused,omitempty"` + Waits string `json:"waits,omitempty"` + VaultName string `json:"vault-name,omitempty"` + Generation int `json:"generation,omitempty"` +} + +// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has +// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key +// asked, said so. +func (h *Holder) outcome(key string, p *provisions.Provision) { + p.Ask, p.Warrant = "", nil + keeps, err := h.keepsEntries(p.Provider.Module) + if err != nil { + p.State, p.Why = provisions.Asked, "the manifest of "+p.Provider.Module+" cannot be read: "+err.Error() + return + } + if !keeps { + h.haveOutcome(p) + return + } + if !slices.Contains(BoundReads, p.Provider.Module) { + p.State, p.Why = provisions.Asked, p.Provider.Module+" keeps its outcome in its entries, which mesh-provisioner's "+ + "manifest does not read yet" + return + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil { + p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "" + return + } + var en entry + if err := json.Unmarshal(e.Value, &en); err != nil { + p.State, p.Why = provisions.Asked, "the provider's entry is not readable" + return + } + switch { + case en.State == provisions.Failed: + p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed") + case en.State == provisions.Waits: + p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person") + case en.State == "" && en.Refused != "": + p.State, p.Why = provisions.Failed, en.Refused + case en.State == "" && en.Waits != "": + p.State, p.Why = provisions.Waits, en.Waits + case en.State != "" && en.State != provisions.Provided: + p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is" + return + default: + p.State, p.Why = provisions.Provided, "" + if en.VaultName != "" || en.Generation > 0 { + p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation} + } + } +} + +// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's +// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is +// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run. +func (h *Holder) haveOutcome(p *provisions.Provision) { + where := p.Provider.Module + " on " + p.Provider.Node + // It makes what the consumer needs only once all it depends on is provided (its credential first). + for _, dep := range p.DependsOn { + state := "not composed yet" + if e, err := h.Store.Get(ProvisionsState, dep); err == nil && e != nil { + if d, err := provisions.Decode(e.Value); err == nil { + state = d.State + } + } + if state != provisions.Provided { + p.State, p.Why = provisions.Asked, "waits for "+dep+" ("+state+") before "+where+" can provide it" + return + } + } + key := p.Provider.Node + "." + p.Provider.Module + e, err := h.Store.Get(HaveRead, key) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet" + return + } + have, err := delivery.DecodeHave(e.Value) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable" + return + } + wantFP := "" + if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil { + if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed { + wantFP = w.FP + } + } + switch { + case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed: + p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed") + case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP): + p.State, p.Why = provisions.Provided, "" + default: + p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+ + "health %s)", where, have.Phase, have.Health) + } +} + +// keepsEntries says a provider's manifest at the trunk declares the state `bound`: its outcome per consumer is its +// entries there; any other provider's is its own have. +func (h *Holder) keepsEntries(module string) (bool, error) { + mk, err := delivery.ModulesKey(module) + if err != nil { + return false, err + } + e, err := h.Store.Get(ModulesRead, mk) + if err != nil || e == nil { + return false, fmt.Errorf("the indexer holds no modules.%s (%v)", module, err) + } + m, err := delivery.DecodeModule(e.Value) + if err != nil || m.Manifest == "" { + return false, fmt.Errorf("the indexer's modules.%s names no manifest", module) + } + key, err := delivery.ManifestsKey(m.Manifest) + if err != nil { + return false, err + } + me, err := h.Store.Get(ManifestsRead, key) + if err != nil || me == nil { + return false, fmt.Errorf("the indexer holds no manifest of %s (%v)", module, err) + } + man, err := delivery.DecodeManifest(me.Value) + if err != nil { + return false, err + } + var decl struct { + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal([]byte(man.JSON), &decl); err != nil { + return false, err + } + for _, raw := range decl.State { + var name string + if json.Unmarshal(raw, &name) != nil { + var full struct { + Name string `json:"name"` + } + _ = json.Unmarshal(raw, &full) + name = full.Name + } + if name == "bound" { + return true, nil + } + } + return false, nil +} + +// retired says a retiring key the operator approved is retired: its provider's entry is gone. +func (h *Holder) retired(key string, p provisions.Provision) (bool, string) { + if p.Warrant == nil || p.Warrant.Option != OptionApprove { + return false, "" + } + if keeps, err := h.keepsEntries(p.Provider.Module); err != nil || !keeps { + return false, "" + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil || e != nil { + return false, "" + } + return true, "its provider removed its credential on the operator's warrant" +} + +// ---- the retiring ask ---------------------------------------------------------------------------------------------- + +// The options a retiring ask offers. The chosen one is the warrant's option in the key, and a provider lets its entry go +// only on "yes" (mesh-sdk provider.go, the pass's retiring keys), so the approving option is "yes" (review of +// 2026-10-30: the holder wrote "approve", which no provider acted on). +const ( + OptionApprove = "yes" + OptionDecline = "no" +) + +// askLasts is how long a retiring ask stays open. +const askLasts = asks.ApproveLasts - 10*time.Minute + +// retiringAsk is the ask of one retiring provision, composed from the key alone. +func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask { + what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node, + p.Consumer.Module, p.Consumer.Node) + a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?", + Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+ + "is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node, + p.Provision, p.Provider.Module, what), + Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes", + About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"}, + {OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"}, + } { + binds, _ := asks.ActDigest(retireAct(key, id, o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +func retireAct(key, id, option string) asks.Act { + return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option} +} + +// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands. +func (h *Holder) ask(p *provisions.Provision, key string) error { + if p.Ask != "" { + return nil + } + if h.Asks == nil { + return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed") + } + now := h.Now().UTC() + id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix()) + a := retiringAsk(key, *p, now, id) + if err := a.Check(now); err != nil { + return err + } + if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil { + return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err) + } + p.Ask = id + return nil +} + +// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the +// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes. +func (h *Holder) Decide() []string { + h.mu.Lock() + defer h.mu.Unlock() + if h.Asks == nil { + return nil + } + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return []string{"the provisions cannot be read: " + err.Error()} + } + var said []string + for _, k := range keys { + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + p, err := provisions.Decode(e.Value) + if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil { + continue + } + raw, found, err := h.Asks.Record(p.Ask) + if err != nil || !found { + continue + } + var rec struct { + State string `json:"state"` + Ask *asks.Ask `json:"ask"` + Warrant *asks.Warrant `json:"warrant"` + } + if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != p.Ask { + continue + } + option := "" + if w.Outcome == asks.OutcomeChosen { + o, err := w.For(Module, *rec.Ask) + if err == nil { + err = o.Performs(retireAct(k, p.Ask, o.ID)) + } + if err != nil { + said = append(said, k+": the answer is refused: "+err.Error()) + continue + } + option = o.ID + } else { + // Ended with no choice: asked again at the next change. + p.Ask = "" + h.put(k, p) + said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes") + continue + } + by := "the operator" + if w.By != nil { + by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) + } + p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()} + h.put(k, p) + said = append(said, k+": "+option+" by "+by) + } + return said +} + +func (h *Holder) put(key string, p provisions.Provision) { + p.At = h.Now().UTC() + if raw, err := provisions.Encode(p); err == nil { + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + h.Logf("[%s] %s is not written: %v", Module, key, err) + } + } +} + +// ---- what a change touches -------------------------------------------------------------------------------------- + +// NodesOfPlacement is the node a placement key names. +func NodesOfPlacement(key string) []string { + if n, err := delivery.ParsePlacementKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfWant is the node a want key names. +func NodesOfWant(key string) []string { + if n, _, err := delivery.ParseWantKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change +// of the module's manifest touches. +func (h *Holder) NodesOfModule(module string) []string { + nodes := map[string]bool{} + for _, n := range h.placedOn(module) { + nodes[n] = true + } + if keys, err := h.Store.Keys(ProvisionsState); err == nil { + for _, k := range keys { + if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil { + if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module { + nodes[p.Consumer.Node] = true + } + } + } + } + return sortedKeys(nodes) +} + +// placedOn is every node a module is placed on, sorted. +func (h *Holder) placedOn(module string) []string { + nodes := map[string]bool{} + if keys, err := h.Store.Keys(PlacementRead); err == nil { + for _, k := range keys { + n, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + e, err := h.Store.Get(PlacementRead, k) + if err != nil || e == nil { + continue + } + if p, err := delivery.DecodePlacement(e.Value); err == nil { + for _, m := range p.Modules { + if m.Module == module { + nodes[n] = true + } + } + } + } + } + return sortedKeys(nodes) +} + +// NodeOfEntry is the consumer's node a provider's entry key names. +func NodeOfEntry(key string) []string { + if n, _, _, _, err := provisions.ParseKey(key); err == nil { + return []string{n} + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A +// retiring key whose provider removed its entry on the operator's warrant goes. +func (h *Holder) Rejudge(key string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + e, err := h.Store.Get(ProvisionsState, key) + if err != nil || e == nil { + return "", err + } + have, err := provisions.Decode(e.Value) + if err != nil { + return "", err + } + if have.State == provisions.Retiring { + if gone, why := h.retired(key, have); gone { + return key + " retired: " + why, h.Store.Delete(ProvisionsState, key) + } + return "", nil + } + next := have + h.outcome(key, &next) + if same(have, next) { + return "", nil + } + h.put(key, next) + return key + " " + next.State, nil +} + +// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at +// the start. +func (h *Holder) RetireNeeds() (int, error) { + keys, err := h.Store.Keys(Needs) + if err != nil { + return 0, err + } + n := 0 + for _, k := range keys { + if err := h.Store.Delete(Needs, k); err != nil { + return n, err + } + n++ + } + return n, nil +} + +// orSaid is the first of the words that says something. +func orSaid(words ...string) string { + for _, w := range words { + if strings.TrimSpace(w) != "" { + return w + } + } + return "" +} + +// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names, read through +// this module's runtime with its own `reads` (frombus.StatesReads). A plan names no value, so no own secret and no +// credential is read (compose.Planning). +func FromBus() Composer { + r := fromruntime.Reader() + return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go new file mode 100644 index 00000000..9aa766c3 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go @@ -0,0 +1,229 @@ +//go:build integration + +package main + +// The holder against a real bus (novox/hq ADR 0570 clauses 1-3, 5; issue 1954 H5): its states and every input it reads +// are a real nats-server's key-value buckets, its plans composed by mesh-sdk's composition from bus state over the +// in-memory mesh frombustest gives (restic on n1, requiring postgres-database from postgres there). A placement and a +// want compose the key; the providers' outcomes (the vault's entry for the credential, postgres's own have) turn it +// provided; restic leaving the node makes it retiring and raises the ask. + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/compose/frombus/frombustest" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// kvStore is the Store over a real bus: this module's states by their local names, another's as `.`. +type kvStore struct { + ctx context.Context + js jetstream.JetStream +} + +func bucketOf(state string) string { + if owner, name, ok := strings.Cut(state, "."); ok { + return owner + "_" + name + } + return Module + "_" + state +} + +func (s kvStore) kv(state string) (jetstream.KeyValue, error) { + kv, err := s.js.KeyValue(s.ctx, bucketOf(state)) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return s.js.CreateKeyValue(s.ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state)}) + } + return kv, err +} + +func (s kvStore) Get(state, key string) (*Entry, error) { + kv, err := s.kv(state) + if err != nil { + return nil, err + } + e, err := kv.Get(s.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &Entry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil +} + +func (s kvStore) Put(state, key string, value any) error { + kv, err := s.kv(state) + if err != nil { + return err + } + raw, err := json.Marshal(value) + if err != nil { + return err + } + _, err = kv.Put(s.ctx, key, raw) + return err +} + +func (s kvStore) Delete(state, key string) error { + kv, err := s.kv(state) + if err != nil { + return err + } + return kv.Delete(s.ctx, key) +} + +func (s kvStore) Keys(state string) ([]string, error) { + kv, err := s.kv(state) + if err != nil { + return nil, err + } + keys, err := kv.Keys(s.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +type asked struct{ ids []string } + +func (a *asked) Publish(_ string, body any, id string) error { + a.ids = append(a.ids, id) + _ = body.(asks.Ask) + return nil +} +func (a *asked) Record(string) ([]byte, bool, error) { return nil, false, nil } + +// 1954/H5. +func TestTheHolderComposesProvidesAndAsksOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + s := kvStore{ctx: ctx, js: js} + mesh, err := frombustest.Provisioned() + if err != nil { + t.Fatal(err) + } + node := frombustest.Node + // The bus as the holder reads it: the indexer's modules and manifests, a placement naming where the vault runs, the + // want of restic on n1. + for module, m := range mesh.Module { + m.V = 1 + raw, _ := json.Marshal(m) + if err := s.Put(ModulesRead, module, json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + } + for digest, raw := range mesh.Manifests { + var named struct { + Module string `json:"module"` + } + _ = json.Unmarshal(raw, &named) + value, _ := delivery.EncodeManifest(delivery.Manifest{Module: named.Module, JSON: string(raw)}) + key, _ := delivery.ManifestsKey(digest) + if err := s.Put(ManifestsRead, key, json.RawMessage(value)); err != nil { + t.Fatal(err) + } + } + // The vault keeps its outcomes as entries: its manifest declares `bound`. + vaultManifest := `{"module":"mesh-vault","version":"1","state":["bound"]}` + vaultDigest := delivery.ManifestDigest([]byte(vaultManifest)) + vaultValue, _ := delivery.EncodeManifest(delivery.Manifest{Module: provisions.VaultModule, JSON: vaultManifest}) + vaultKey, _ := delivery.ManifestsKey(vaultDigest) + if err := s.Put(ManifestsRead, vaultKey, json.RawMessage(vaultValue)); err != nil { + t.Fatal(err) + } + if err := s.Put(ModulesRead, provisions.VaultModule, map[string]any{"v": 1, "none2": "the test's", "manifest": vaultDigest, + "repository": "novox/mesh-catalog", "commit": "c1"}); err != nil { + t.Fatal(err) + } + vault, _ := json.Marshal(map[string]any{"v": 1, "node": "n2", "decided": map[string]any{}, + "modules": []map[string]any{{"module": provisions.VaultModule}}}) + if err := s.Put(PlacementRead, "placement.n2", json.RawMessage(vault)); err != nil { + t.Fatal(err) + } + fp := frombustest.FPs()["restic"] + if err := s.Put(WantsRead, "want."+node+".restic", map[string]any{"v": 1, "fp": fp, "self": true}); err != nil { + t.Fatal(err) + } + a := &asked{} + h := &Holder{Store: s, Asks: a, Now: time.Now, Logf: t.Logf, + Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + return frombus.Plans(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + }} + state := func(key string) provisions.Provision { + e, _ := s.Get(ProvisionsState, key) + if e == nil { + return provisions.Provision{} + } + p, _ := provisions.Decode(e.Value) + return p + } + key := node + ".restic.postgres-database" + if _, err := h.ComposeNode(ctx, node); err != nil { + t.Fatal(err) + } + if p := state(key); p.State != provisions.Asked || p.Provider.Module != "postgres" { + t.Fatalf("the placement and the want composed %+v", p) + } + if e, _ := s.Get(PlansState, node+".restic"); e == nil { + t.Fatal("restic's plan is not on the bus") + } + // The providers' outcomes: the vault provides the credential where restic has one, postgres runs healthy. + for _, dep := range state(key).DependsOn { + if err := s.Put(boundRead(provisions.VaultModule), dep, map[string]any{"state": "provided"}); err != nil { + t.Fatal(err) + } + } + if err := s.Put(HaveRead, node+".postgres", map[string]any{"v": 1, "fp": frombustest.FPs()["postgres"], "phase": "running", + "health": "ok"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { // the credential judged first, then what depends on it + if _, err := h.ComposeNode(ctx, node); err != nil { + t.Fatal(err) + } + } + if p := state(key); p.State != provisions.Provided { + t.Fatalf("the outcomes did not turn it provided: %+v", p) + } + // restic leaves the node and its want goes: retiring, one ask. + p := mesh.Placement[node] + var kept []delivery.PlacedModule + for _, m := range p.Modules { + if m.Module != "restic" { + kept = append(kept, m) + } + } + p.Modules = kept + mesh.Placement[node] = p + if err := s.Delete(WantsRead, "want."+node+".restic"); err != nil { + t.Fatal(err) + } + if _, err := h.ComposeNode(ctx, node); err != nil { + t.Fatal(err) + } + if r := state(key); r.State != provisions.Retiring || r.Ask == "" || len(a.ids) == 0 { + t.Fatalf("removing the want did not ask: %+v, asked %v", r, a.ids) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go new file mode 100644 index 00000000..238b589c --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -0,0 +1,362 @@ +package main + +import ( + "context" + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// router is the operator's channel as the holder asks on it. +type router struct { + asked map[string]asks.Ask + records map[string][]byte +} + +func (r *router) Publish(_ string, body any, id string) error { + r.asked[id] = body.(asks.Ask) + return nil +} + +func (r *router) Record(id string) ([]byte, bool, error) { + raw, ok := r.records[id] + return raw, ok, nil +} + +// answer is the operator's answer to an ask. +func (r *router) answer(t *testing.T, id, option string) { + t.Helper() + a := r.asked[id] + o, ok := a.Option(option) + if !ok { + t.Fatalf("the ask %s offers no %s", id, option) + } + w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, + Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram", + At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()} + r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w}) +} + +// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted +// builds and at the newest. +type mesh struct { + h *Holder + s *memStore + r *router + atWant map[string]map[string]catalogue.Plan + atNewest map[string]map[string]catalogue.Plan + composed []string +} + +func newMesh() *mesh { + m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}}, + atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}} + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + m.composed = append(m.composed, node) + if keptAt != nil { + return m.atWant[node], nil + } + return m.atNewest[node], nil + }} + // What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their + // have. + for module, states := range map[string]string{"postgres": `["bound"]`, "keycloak": `["bound"]`, "mesh-vault": `["bound"]`, + "systemd-resolved": `[]`, "minio": `[]`} { + raw := `{"module":"` + module + `","version":"1","state":` + states + `}` + digest := delivery.ManifestDigest([]byte(raw)) + value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw}) + _ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value)) + _ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)), + json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`)) + } + return m +} + +func mustKey(k string, err error) string { + if err != nil { + panic(err) + } + return k +} + +// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set). +func planOf(provs ...catalogue.PlanProvision) catalogue.Plan { + return catalogue.Plan{Provisions: provs} +} + +var ( + pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"} + oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"} + split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"} +) + +func (m *mesh) state(t *testing.T, key string) provisions.Provision { + t.Helper() + e, _ := m.s.Get(ProvisionsState, key) + if e == nil { + return provisions.Provision{} + } + p, err := provisions.Decode(e.Value) + if err != nil { + t.Fatal(err) + } + return p +} + +func (m *mesh) compose(t *testing.T, node string) Composed { + t.Helper() + c, err := m.h.ComposeNode(context.Background(), node) + if err != nil { + t.Fatal(err) + } + return c +} + +// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry +// alone; the vault's generation reaches the reference; a +// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2). +func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { + m := newMesh() + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)} + m.atNewest["g14"] = m.atWant["g14"] + key := "g14.gitea.postgres-database" + steps := []struct { + name string + entry string // the provider's entry written before the step, "-" to delete + state string + why string + }{ + {"composed: asked", "", provisions.Asked, ""}, + {"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"}, + {"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"}, + {"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"}, + {"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""}, + } + for _, s := range steps { + if s.entry != "" { + m.s.write(t, "postgres.bound", key, s.entry) + } + m.compose(t, "g14") + p := m.state(t, key) + if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" { + t.Errorf("%s: %+v", s.name, p) + } + } + if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 { + t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference) + } + // A provider making no credential per consumer reports through its own have (clause 3): never provided by the + // holder alone. + split := "g14.gitea.split-dns" + for _, row := range []struct{ have, want, state, why string }{ + {"", "", provisions.Asked, "says nothing yet"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""}, + } { + if row.have != "" { + m.s.write(t, HaveRead, "g14.systemd-resolved", row.have) + } + if row.want != "" { + m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want) + } + m.compose(t, "g14") + if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) { + t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state) + } + } + if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { + t.Error("the plan is not written") + } + if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 { + t.Errorf("a quiet pass wrote %+v", c) + } + // A provision's credential is the vault's own provision, on the consumer's node for a login of its own, and the + // provision depends on it (clauses 3, 6). + c := newMesh() + c.s.write(t, PlacementRead, "placement.novox", `{"v":1,"node":"novox","decided":{},"modules":[{"module":"mesh-vault"}]}`) + pair := pgDB + pair.CredentialForm = "pair" + c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pair)} + c.atNewest["g14"] = c.atWant["g14"] + c.compose(t, "g14") + cred := c.state(t, "g14.gitea.secret-postgres-database") + if cred.Provider.Module != "mesh-vault" || cred.Provider.Node != "novox" || cred.State != provisions.Asked { + t.Errorf("the credential's provision is %+v", cred) + } + if d := c.state(t, key).DependsOn; !slices.Equal(d, []string{"g14.gitea.secret-postgres-database"}) { + t.Errorf("the provision depends on %v", d) + } + // A pair provider applying the vault's value through its templates (no entries) is provided only once its credential + // key is provided and it runs healthy (the lead's refinement of clause 3). + bucket := catalogue.PlanProvision{Provision: "s3-bucket", Provider: "g14", ProviderModule: "minio", CredentialForm: "pair"} + c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(bucket)} + c.atNewest["g14"] = c.atWant["g14"] + c.s.write(t, HaveRead, "g14.minio", `{"v":1,"fp":"src2:`+strings.Repeat("cc", 32)+`","phase":"running","health":"ok"}`) + s3, s3cred := "g14.gitea.s3-bucket", "g14.gitea.secret-s3-bucket" + for _, row := range []struct { + name, entry, cred, state string + }{ + {"its credential still asked", "", provisions.Asked, provisions.Asked}, + {"its credential provided by the vault", `{"state":"provided","vault-name":"pair/g14/s3-bucket/g14/gitea","generation":1}`, + provisions.Provided, provisions.Provided}, + } { + if row.entry != "" { + c.s.write(t, "mesh-vault.bound", s3cred, row.entry) + } + c.compose(t, "g14") + c.compose(t, "g14") // the credential judged first, then what depends on it + if got, dep := c.state(t, s3), c.state(t, s3cred); got.State != row.state || dep.State != row.cred { + t.Errorf("%s: %s is %s (%s), its credential %s", row.name, s3, got.State, got.Why, dep.State) + } + } + // An entry's change judges its key alone, no composition. + before := len(m.composed) + m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`) + if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before { + t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before) + } +} + +// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed +// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the +// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again; +// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3). +func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { + m := newMesh() + pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client" + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want + m.s.write(t, "postgres.bound", pg, `{}`) + steps := []struct { + name string + change func() + asked []string + states map[string]string + removed bool + }{ + {"a key composed ahead of its want is asked, not retiring", func() {}, nil, + map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false}, + {"one requirement removed and another added: only the removed retires", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + }, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false}, + {"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false}, + {"needed again before the warrant: back to its outcome", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)} + }, nil, map[string]string{pg: provisions.Provided}, false}, + {"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} }, + []string{pg}, map[string]string{pg: provisions.Retiring}, false}, + } + for _, s := range steps { + s.change() + c := m.compose(t, "g14") + if !slices.Equal(c.Asked, s.asked) { + t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked) + } + for k, st := range s.states { + if p := m.state(t, k); p.State != st { + t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why) + } + } + } + p := m.state(t, pg) + a := m.r.asked[p.Ask] + if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") || + !strings.Contains(a.Explanation, "data stay") { + t.Errorf("the ask does not say what goes from which provider on which node: %+v", a) + } + // Nothing before the warrant: the provider's entry stands and the key stays. + if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil { + t.Errorf("decided before an answer: %v", said) + } + m.r.answer(t, p.Ask, OptionApprove) + m.h.Decide() + if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") { + t.Fatalf("the warrant is not in the key: %+v", w) + } + // The word the SDK's provider lets an entry go on (provider.go): any other, and no provider ever retires it. + if w := m.state(t, pg).Warrant; w.Option != "yes" { + t.Fatalf("the warrant says %q, and a provider lets go only on \"yes\"", w.Option) + } + if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring { + t.Errorf("retired while the provider's entry stands: %q", said) + } + _ = m.s.Delete("postgres.bound", pg) + if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" { + t.Errorf("not retired once the provider removed its entry: %q", said) + } + // A decline is kept, and nothing is asked again while nothing changes. + d := newMesh() + d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + d.compose(t, "g14") + d.atWant["g14"] = map[string]catalogue.Plan{} + d.compose(t, "g14") + d.r.answer(t, d.state(t, pg).Ask, OptionDecline) + d.h.Decide() + if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline { + t.Errorf("a decline was asked again: %+v", c) + } +} + +// Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build +// its have says it runs, so the holder writes it a plan and its client keeps filling it. +func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { + m := newMesh() + fp := "src2:" + strings.Repeat("dd", 32) + m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`) + m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`) + var keptAt map[string]string + m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) { + if at != nil { + keptAt = at + return map[string]catalogue.Plan{"nats": {}}, nil + } + return nil, nil + } + c := m.compose(t, "novox") + if keptAt["nats"] != fp || len(keptAt) != 1 { + t.Fatalf("composed at %v, want nats at the build it runs and nothing of another node", keptAt) + } + if !slices.Contains(c.Plans, "novox.nats") { + t.Fatalf("no plan written for a module held without a want: %+v", c) + } +} + +// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1). +func TestTheHolderHoldsNoTimer(t *testing.T) { + files, _ := filepath.Glob("*.go") + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" && + slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) { + t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name) + } + } + return true + }) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go new file mode 100644 index 00000000..30a16446 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -0,0 +1,104 @@ +package main + +// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and +// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The holder's tools. +const ( + ProvisionsTool = "provisioner_provisions" + PlansTool = "provisioner_plans" + ComposeTool = "provisioner_compose" + DecideTool = "provisioner_decide" +) + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +func holderTools(h *Holder) []stdio.Tool { + arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: ProvisionsTool, + Description: "Every provision the seat keeps (novox/hq ADR 0570): its key ..[.], " + + "provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " + + "state, with how many keys and bytes the state holds; or one key whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"), + "node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}}, + Run: func(a map[string]any) (any, error) { + return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state")) + }}, + {Name: PlansTool, + Description: "Every module's provisioning plan the seat composed, ., narrowed by node, with how many " + + "keys and bytes the state holds; or one plan whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, ."), + "node": str("a node")}}, + Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }}, + {Name: ComposeTool, + Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " + + "was written, removed, changed and asked.", + Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")}, + "required": []string{"node"}}, + Run: func(a map[string]any) (any, error) { + if arg(a, "node") == "" { + return nil, fmt.Errorf("compose names the node") + } + return h.ComposeNode(context.Background(), arg(a, "node")) + }}, + {Name: DecideTool, + Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " + + "key, as the router's word on an ask would.", + Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }}, + } +} + +// read is a state's keys narrowed, or one whole, with the state's fill. +func (h *Holder) read(state, key, node, want string) (any, error) { + if key != "" { + e, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("%s holds no %s", state, key) + } + return json.RawMessage(e.Value), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := []map[string]any{} + bytes := 0 + for _, k := range keys { + e, err := h.Store.Get(state, k) + if err != nil || e == nil { + continue + } + bytes += len(e.Value) + if node != "" && !strings.HasPrefix(k, node+".") { + continue + } + row := map[string]any{"key": k} + if state == ProvisionsState { + p, err := provisions.Decode(e.Value) + if err != nil { + row["unreadable"] = err.Error() + } else { + if want != "" && p.State != want { + continue + } + row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why + } + } + rows = append(rows, row) + } + return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 3955516d..cca4a622 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -1,23 +1,28 @@ // mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR -// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR -// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its -// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its -// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this -// module says, it says on stderr. +// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259 +// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change +// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and +// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its +// start. stdout is the MCP channel; what this module says, it says on stderr. package main import ( + "context" "fmt" "os" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. const ( - Seat = "mesh-provisioner" - Verb = "provisioning" + Seat = "mesh-provisioner" + Module = "mesh-provisioner" + Verb = "provisioning" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state). func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the +// router's record of one read under that name. +type runtimeAsks struct{} + +func (runtimeAsks) Publish(subject string, body any, id string) error { + return stdio.SeatPublish(subject, body, id) +} + +func (runtimeAsks) Record(id string) ([]byte, bool, error) { + e, err := stdio.SeatRecord("", id) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake. +type work struct { + mu sync.Mutex + pending map[string]bool + wake chan struct{} +} + +func (w *work) add(nodes ...string) { + if len(nodes) == 0 { + return + } + w.mu.Lock() + for _, n := range nodes { + w.pending[n] = true + } + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *work) run(h *Holder) { + for range w.wake { + w.mu.Lock() + nodes := sortedKeys(w.pending) + w.pending = map[string]bool{} + w.mu.Unlock() + for _, n := range nodes { + c, err := h.ComposeNode(context.Background(), n) + switch { + case err != nil: + logf("[%s] %s is not composed: %v", Module, n, err) + case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0: + logf("[%s] composed %s: %+v", Module, n, c) + } + } + } +} + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch("", handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + func main() { - go publishing(runtimeStore{}) - if err := stdio.Serve("", Tools(runtimeStore{})); err != nil { + s := runtimeStore{} + h := &Holder{Store: s, Compose: FromBus(), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} + w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)} + go w.run(h) + go starting(h) + go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil }) + go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil }) + go watched(ModulesRead, func(c stdio.StateChange) error { + if module, err := delivery.ParseModulesKey(c.Key); err == nil { + w.add(h.NodesOfModule(module)...) + } + return nil + }) + for _, provider := range BoundReads { + go watched(boundRead(provider), func(c stdio.StateChange) error { + if said, err := h.Rejudge(c.Key); err != nil { + logf("[%s] %s is not judged again: %v", Module, c.Key, err) + } else if said != "" { + logf("[%s] %s", Module, said) + } + return nil + }) + } + go watched(Registry, func(c stdio.StateChange) error { + if c.Op != "put" { + if _, _, err := Publish(s); err != nil { + logf("[%s] the registry is not written back: %v", Module, err) + } + } + return nil + }) + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error { + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) + } + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait) + } + }() + if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil { logf("%v", err) os.Exit(1) } } -// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or -// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first -// pass is tried again until it lands. -func publishing(s Store) { - for wait := 2 * time.Second; ; { - time.Sleep(wait) - written, deleted, err := Publish(s) - switch { - case err != nil: - logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait) - wait = min(wait*2, time.Minute) - continue - case len(written) > 0 || len(deleted) > 0: - logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted) +// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers, +// each tried again until it lands. +func starting(h *Holder) { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + written, deleted, err := Publish(h.Store) + if err == nil { + if len(written)+len(deleted) > 0 { + logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted) + } + break } - wait = time.Hour + logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait) + time.Sleep(wait) + } + if n, err := h.RetireNeeds(); err != nil { + logf("[%s] the retired needs are not all deleted: %v", Module, err) + } else if n > 0 { + logf("[%s] deleted %d key(s) of the retired needs", Module, n) + } + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index 4661b226..dd0ccde3 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -8,6 +8,8 @@ import ( "strings" "sync" "testing" + + "git.novox.be/novox/mesh-sdk/go/compose/frombus" ) // memStore is the bus as mesh-provisioner reaches it, in memory: the states it keeps. @@ -106,9 +108,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) { for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect, "shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect, "contribution": "mesh-provisioner_" + Collect, - "dir": "mesh-controller_assignments", "access": "mesh-controller_assignments", - "port": "mesh-controller_assignments", "seat": "mesh-controller_assignments", - "machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} { + "dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans", + "port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans", + "machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} { i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind }) if TheRegistry[i].Bucket != bucket { t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket) @@ -288,6 +290,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { Serves []string `json:"serves"` } `json:"claims"` State []string `json:"state"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` OwnSecrets map[string]string `json:"own-secrets"` } if err := json.Unmarshal(raw, &m); err != nil { @@ -300,7 +304,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { !slices.Equal(m.Claims[0].Serves, []string{Verb}) { t.Errorf("the claim: %+v", m.Claims) } - if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) { + // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } + // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools + // its own and the composition it does on a change. + served := map[string]bool{} + for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) { + served[tool.Name] = true + } + for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb, + Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool, + "a composition on a change": ComposeTool, "a decision on an answer": DecideTool} { + if !served[tool] { + t.Errorf("%s has no tool: %s is not served", what, tool) + } + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which is not served", tool) + } + } + for _, r := range append([]string{WantsRead}, frombus.StatesReads...) { // what fromruntime.Reader reads + if !slices.Contains(m.Reads, r) { + t.Errorf("the manifest does not read %s, which the composition reads", r) + } + } + for _, p := range BoundReads { + if !slices.Contains(m.Reads, boundRead(p)) { + t.Errorf("the manifest does not read %s's entries", p) + } + } + if slices.Contains(m.Reads, "mesh-controller.assignments") { + t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)") + } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside new file mode 100644 index 00000000..203aee31 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-provisioner/module.json diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go index 7d7d6f3e..15f0bb3f 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go @@ -48,16 +48,15 @@ type Answer struct { Decision string `json:"decision"` } -// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's -// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which -// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in -// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration, -// which the node-engine keeps for root alone. +// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR +// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state +// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for +// root alone, nor the controller's assignments. func placement(kind, what string) Answer { - return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments", + return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState, Key: ".", - Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." + - "placement)", Decision: "novox/hq ADR 0349 clause 6"} + Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " + + "(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"} } // oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1). diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index 9f0b6d60..db34240f 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -2,4 +2,21 @@ module mesh-provisioner go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index bb6beaa1..a2b8e018 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,2 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 650425b3..1aba09af 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -11,11 +11,36 @@ ] } ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-assigner.placement", + "mesh-delivery-assigner.wants", + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.manifests", + "mesh-delivery-builder.builds", + "node-engine.have", + "mesh-config.resolved", + "gitea.bound", + "keycloak.bound", + "postgres.bound", + "route-adapter.bound", + "mesh-vault.bound" + ], + "tools": [ + "provisioner_provisions", + "provisioner_plans", + "provisioner_compose", + "provisioner_decide" + ], "state": [ "registry", "needs", "applied", - "collect" + "collect", + "provisions", + "plans" ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/modules/mesh-self-check/.gitignore b/modules/mesh-self-check/.gitignore new file mode 100644 index 00000000..2c7bac8b --- /dev/null +++ b/modules/mesh-self-check/.gitignore @@ -0,0 +1,2 @@ +/mesh-self-check +/cmd/mesh-self-check/mesh-self-check diff --git a/modules/mesh-self-check/README.md b/modules/mesh-self-check/README.md new file mode 100644 index 00000000..9411474c --- /dev/null +++ b/modules/mesh-self-check/README.md @@ -0,0 +1,33 @@ +# mesh-self-check + +Holds the seat `mesh-self-check` (novox/hq ADR 0564 §6): the mesh's self-check, moved out of the controller. Go, one +binary (`cmd/mesh-self-check`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes its buckets. + +## What it does + +Every five minutes (the first run a minute after it starts) it runs its registry of probes, each within thirty +seconds. A probe is one live invariant of a design, with its assertion as the controller had it, reading its facts on +the bus (the bus module's tools for the bus's own state). A probe whose facts have no home on the bus yet answers +`deferred`, naming them: never a pass. A probe that cannot run is said in the verdict at once, and written as a finding +of its own (`probe..failed`) when the next run cannot run it either. + +What a probe finds is written as a finding, one key `..` per fault, only when what it says changes, and +deleted when a run no longer sees it. The condition keeper (`mesh-conditions`) raises and clears conditions from them. +A finding one look can be wrong about (a consumer behind after a burst) is written when two runs in a row see it. + +Every run ends with the event `checked`, the body of the controller's `doctor-heartbeat`, which the watchman (S10) and +mesh-watcher on another machine hear: a self-check that stops checking is said through a path that does not pass +through it. + +## What it keeps + +- `mesh-self-check_findings`: what it finds wrong now. +- `mesh-self-check_verdict`: the key `last`, the last run whole. +- `mesh-self-check_verdicts`: every run (each probe's findings at most twenty), kept a week by the bucket's age. + +## Its verbs and tools + +- `mesh-self-check.doctor`: the last verdict; `run=true` runs now; `probe=` one probe whole. +- `mesh-self-check.probes`: the registry, with what each deferred probe lacks. +- `self_check_findings`, `self_check_verdicts`: the reads of its buckets. diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go new file mode 100644 index 00000000..6d3d53f8 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -0,0 +1,74 @@ +// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds +// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts` +// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is +// the MCP channel; what this module says, it says on stderr. +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// NewChecker is the self-check over the runtime. +func NewChecker() *Checker { + return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), + Enabled: enabledProbes, + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} +} + +func main() { + c := NewChecker() + go func() { + <-stdio.Served() + c.Keep(context.Background()) + }() + if err := stdio.Serve("", Tools(c)); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// enabledProbes is the setting `probes`, read at each run so a change applies at the next; with it unreadable, the +// default list. +func enabledProbes() []string { + raw, err := stdio.Config("probes") + if err != nil { + logf("the setting probes cannot be read (%v); the default list runs", err) + return DefaultProbes + } + var list []string + if json.Unmarshal(raw, &list) != nil { + logf("the setting probes is not a list of probe ids (%s); the default list runs", raw) + return DefaultProbes + } + return list +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go new file mode 100644 index 00000000..e288898c --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves +// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts, +// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by +// itself a tool that starts it by hand (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + m := readManifest(t) + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + // checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 { + t.Errorf("emits %v", m.Emits) + } + if !slices.Equal(m.Invokes, Invokes) { + t.Errorf("invokes %v, want what the probes ask, %v", m.Invokes, Invokes) + } + if !slices.Equal(m.Reads, Reads) { + t.Errorf("reads %v, want what the probes read, %v", m.Reads, Reads) + } + states := map[string]int{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + TTL int `json:"ttl-seconds"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.TTL + } + if states[VerdictsState] != 7*24*60*60 { + t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState]) + } + // ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool, + "the run": DoctorTool + " run=true"} + tools := map[string]bool{} + for _, tool := range Tools(&Checker{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + name, _, _ := cutSpace(tool) + if !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} + +func cutSpace(s string) (string, string, bool) { + for i, r := range s { + if r == ' ' { + return s[:i], s[i+1:], true + } + } + return s, "", false +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/observation.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go new file mode 100644 index 00000000..71209ce9 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/observation.go @@ -0,0 +1,142 @@ +package main + +// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an +// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong +// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key +// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is the probe that saw it, or the self-check itself for a probe that could not run. + Source string + // Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277). + Confirm bool +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} + +// worded gives a finding the operator's words registered for its kind, where it says none of its own. +func worded(f Observation) Observation { + w, ok := words[f.Kind] + if !ok { + return f + } + if f.Headline == "" { + f.Headline = w.Headline + } + if f.Explanation == "" { + f.Explanation = w.Explanation + } + if f.Resolved == "" { + f.Resolved = w.Resolved + } + if f.Needs == "" { + f.Needs = w.Needs + } + return f +} + +type plain struct{ Headline, Explanation, Resolved, Needs string } + +// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the +// controller's name for itself replaced by the self-check's where the words spoke of it. +var words = map[string]plain{ + "consumer-lost": {Headline: "A listener on the bus is gone", + Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", + Resolved: "The listener is back"}, + "consumer-wrong": {Headline: "A listener on the bus is set up wrong", + Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.", + Resolved: "The listener is set up right again"}, + "consumer-behind": {Headline: "Messages pile up for a listener", + Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " + + "restarts it if it stays.", + Resolved: "The listener caught up"}, + "stream-wrong": {Headline: "Part of the bus's storage is wrong", + Needs: "check the machine the bus runs on; the details say what is missing.", + Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", + Resolved: "The bus's storage is right again"}, + "bus-reconnects": {Headline: "A client keeps losing the bus", + Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + + "it reconnects, what it says and what it is asked waits.", + Resolved: "Resolved: the client stays connected"}, + "core-unhealthy": {Headline: "The bus is not healthy", + Needs: "check the machine the bus runs on; the details say what fails.", + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + + "calls may fail until it recovers.", + Resolved: "The bus is healthy again"}, + "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", + Needs: "restart the watchman if this stays, from the mesh MCP server.", + Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + + "signals are not noticed.", + Resolved: "The watchdogs run again"}, + "probe-failed": {Headline: "A check of the mesh could not run", + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + + "every few minutes.", + Resolved: "The check runs again"}, +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go new file mode 100644 index 00000000..e4dbce45 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -0,0 +1,453 @@ +package main + +// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller, +// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home +// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the +// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the +// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and +// the bus step (its home is issue 1953's). + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat. +const ( + StreamsTool = "nats.nats_streams" + ClosedTool = "nats.nats_closed_connections" + SignalsVerb = "seat:mesh-watchdogs.signals" + notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; " + storeOnly = "held only in the controller's store today" + behindBound = 1000 + reconnectMax = 12 +) + +// Invokes are the tools and seat verbs the probes ask, and Reads the states they read: the manifest's `invokes` and +// `reads`, which its test holds to these. +var ( + Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb} + Reads = []string{PlacementRead, SeatHoldersRead, PresenceRead, HealthRead, ReadingsRead, GoodRead, ModulesRead, ManifestsRead} +) + +// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe +// that moves here is a row added, and the registry test holds it to ADR 0564's table. +func Registry() []Probe { + return []Probe{ + {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", + From: "issue 262", Kind: "resolver-wrong", + Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly}, + {ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " + + "is heard from", From: "issues 208, 218", Kind: "holder-silent", + Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " + + "records are " + storeOnly + ", and the watchman's presence has no read granted to this module"}, + {ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld", + Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly}, + {ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " + + "stream's head", From: "issues 248, 266", Kind: "consumer-wrong", + Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers}, + {ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208", + Kind: "stream-wrong", Run: probeStreams}, + {ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list", + From: "issue 238", Kind: "own-address-banned", Run: probeBans}, + {ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " + + "cleanup", From: "ADR 0230", Kind: "cleanup-waiting", + Deferred: "the retirement records of providers' consumers: " + storeOnly}, + {ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " + + "by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved", + Deferred: "the bindings of provisions to their consumers: " + storeOnly}, + {ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " + + "backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank", + Raises: []string{kindDataMissing, kindDataQuiet, kindBackupStale, kindArrayDegraded, kindProtectionMissing, + kindDataUnmeasured}, + Run: probeData}, + {ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " + + "bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects}, + {ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " + + "statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8", + Kind: "agent-can-become-root", Run: probeAgentAccounts}, + {ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " + + "proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free", + Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly}, + {ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " + + "filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501", + Kind: "bucket-or-log-filling", Run: probeFill}, + {ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it", + From: "ADR 0376 §2", Kind: "grant-composes-nothing", + Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly}, + {ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals", + From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs}, + {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " + + "to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth}, + } +} + +// --- the bus's own state, as its module answers it --------------------------------------------------------------- + +// streamsAnswer is what nats_streams answers. +type streamsAnswer struct { + Detail []streamRow `json:"detail"` +} + +type streamRow struct { + Name string `json:"name"` + Subjects []string `json:"subjects"` + Retention string `json:"retention"` + MaxPerSubject int64 `json:"max_per_subject"` + Consumers []consumerRow `json:"consumer_detail"` +} + +type consumerRow struct { + Name string `json:"name"` + Kind string `json:"kind"` + Filters []string `json:"filters"` + Pending int64 `json:"pending"` + Unacked int64 `json:"unacknowledged"` + MaxDeliver int64 `json:"max_deliver"` +} + +// errNoStream is a stream the bus module says it does not hold. +var errNoStream = errors.New("no such stream") + +// streams asks the bus module for every stream, or for one with its consumers. +func (c *Checker) streams(stream string) ([]streamRow, error) { + args := map[string]any{} + if stream != "" { + args["stream"] = stream + } + raw, err := c.Ask(StreamsTool, args) + if err != nil { + if stream != "" && strings.Contains(err.Error(), "no stream called") { + return nil, errNoStream + } + return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) + } + var a streamsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err) + } + return a.Detail, nil +} + +// consumerWords is a consumer as the mesh says it, with its name. +func consumerWords(c buscompose.Consumer) string { + return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream) +} + +// expectedConsumers are the durable consumers the mesh defines, by stream. +func expectedConsumers() map[string][]buscompose.Consumer { + out := map[string][]buscompose.Consumer{} + for _, c := range buscompose.MeshConsumers() { + out[c.Stream] = append(out[c.Stream], c) + } + return out +} + +// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. +func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) { + var out []Observation + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if ctx.Err() != nil { + return nil, ctx.Err() + } + rows, err := c.streams(stream) + held := map[string]consumerRow{} + switch { + case errors.Is(err, errNoStream): + case err != nil: + return nil, err + default: + for _, r := range rows { + for _, cr := range r.Consumers { + held[cr.Name] = cr + } + } + } + for _, want := range byStream[stream] { + who := want.Stream + "." + want.Name + have, ok := held[want.Name] + if !ok { + out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), + Said: "no consumer " + want.Name + " on " + want.Stream}) + continue + } + if differs := consumerDiffers(want, have); differs != "" { + out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), + Said: differs}) + } + if want.Stream != "NODES" && have.Pending > behindBound { + // One look at a consumer a burst put behind is not one that stays behind: held for the next run, + // so healer H4 resets nothing on a single sample (novox/hq issue 277). + out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), + Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) + } + } + } + return sorted(out), nil +} + +// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its +// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is. +func consumerDiffers(want buscompose.Consumer, have consumerRow) string { + var differs []string + haveFilters := slices.Clone(have.Filters) + wantFilters := slices.Clone(want.Filters) + sort.Strings(haveFilters) + sort.Strings(wantFilters) + if !slices.Equal(haveFilters, wantFilters) { + differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters)) + } + if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) { + differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver)) + } + if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush { + differs = append(differs, "delivers by the other shape (push or pull) than defined") + } + return strings.Join(differs, "; ") +} + +// probeStreams is D7: every stream the mesh defines exists as defined. +func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { + rows, err := c.streams("") + if err != nil { + return nil, err + } + held := map[string]streamRow{} + for _, r := range rows { + held[r.Name] = r + } + var out []Observation + for _, s := range buscompose.MeshStreams() { + have, ok := held[s.Name] + if !ok { + out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), + Said: "no stream " + s.Name}) + continue + } + if differs := streamDiffers(s, have); differs != "" { + out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), + Said: differs}) + } + } + return sorted(out), nil +} + +// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its +// subjects, its retention and how many it keeps per subject. Empty when nothing is. +func streamDiffers(want buscompose.Stream, have streamRow) string { + var differs []string + haveSubjects := slices.Clone(have.Subjects) + wantSubjects := slices.Clone(want.Subjects) + sort.Strings(haveSubjects) + sort.Strings(wantSubjects) + if !slices.Equal(haveSubjects, wantSubjects) { + differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects)) + } + retention, perSubject := "limits", int64(want.MaxMsgsPerSubject) + switch want.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + perSubject = 1 + } + if have.Retention != "" && have.Retention != retention { + differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention)) + } + if perSubject != 0 && have.MaxPerSubject != perSubject { + differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject)) + } + return strings.Join(differs, "; ") +} + +// --- D15 ----------------------------------------------------------------------------------------------------- + +// closedAnswer is what nats_closed_connections answers. +type closedAnswer struct { + Hours float64 `json:"hours"` + Reaches bool `json:"reaches"` + Users []struct { + User string `json:"user"` + Closed int `json:"closed"` + Dropped int `json:"dropped"` + Names []struct { + Name string `json:"name"` + Dropped int `json:"dropped"` + Reasons map[string]int `json:"reasons"` + } `json:"names"` + } `json:"users"` +} + +// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. +func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { + raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) + if err != nil { + return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) + } + var read closedAnswer + if err := json.Unmarshal(raw, &read); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err) + } + hours := read.Hours + if hours <= 0 { + hours = 1 + } + var out []Observation + for _, u := range read.Users { + if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { + continue // refused before it logged in is nobody's reconnect loop + } + reasons := map[string]int{} + var names []string + for _, n := range u.Names { + if n.Dropped == 0 { + continue + } + names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped)) + for r, k := range n.Reasons { + if r != "Client Closed" { + reasons[r] += k + } + } + } + var why []string + for r, k := range reasons { + why = append(why, fmt.Sprintf("%s ×%d", r, k)) + } + sort.Strings(why) + partial := "" + if !read.Reaches { + partial = " (at least: the server's record of closed connections does not reach back the whole hour)" + } + who, machine := busUserWords(u.User) + out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, + Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ + "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), + Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, + hours, strings.Join(names, ", "), strings.Join(why, ", ")), + Headline: capital(who) + " keeps losing the bus", + Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+ + "again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)), + Resolved: "Resolved: " + who + " stays connected"}) + } + return sorted(out), nil +} + +// busUserWords is a bus user as the operator says it, and the machine it is on. +func busUserWords(user string) (string, string) { + switch { + case user == "controller": + return "the controller", "" + case strings.HasPrefix(user, "node."): + m := strings.TrimPrefix(user, "node.") + return "the node-engine on " + m, m + } + if m, module, ok := strings.Cut(user, "."); ok { + if module == "node-tools" { + return "the tool runner on " + m, m + } + return module + " on " + m, m + } + return "a client of the bus", "" +} + +// --- DW and H-bus -------------------------------------------------------------------------------------------- + +// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the +// watchman's pulse on `checked`, is the other half. +func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { + if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { + return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + + "late signal is being said", + Said: firstLine(err.Error())}}, nil + } + return nil, nil +} + +// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it +// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip, +// and no answer is no pass. +func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { + rows, err := c.streams("") + if err != nil { + return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) + } + held := map[string]bool{} + for _, r := range rows { + held[r.Name] = true + } + var problems []string + for _, s := range buscompose.MeshStreams() { + if !held[s.Name] { + problems = append(problems, "the stream "+s.Name+" is missing") + } + } + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if !held[stream] { + continue // said above + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + detail, err := c.streams(stream) + if err != nil && !errors.Is(err, errNoStream) { + return nil, err + } + have := map[string]bool{} + for _, r := range detail { + for _, cr := range r.Consumers { + have[cr.Name] = true + } + } + for _, want := range byStream[stream] { + if !have[want.Name] { + problems = append(problems, consumerWords(want)+" is missing") + } + } + } + if len(problems) == 0 { + return nil, nil + } + return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil +} + +// sorted is findings in a stated order, so two runs over one mesh say the same. +func sorted(out []Observation) []Observation { + sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) + return out +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go new file mode 100644 index 00000000..98d8146f --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go @@ -0,0 +1,392 @@ +package main + +// D13, the data the modules declare (novox/hq ADR 0233, ADR 0571 clause 3, ADR 0564 §6): every item of data a machine +// declares is measured, is there, holds what it held, and is backed up within its bound. The controller composed what +// every machine declares and asked each machine's backup holder what it measured; the facts now live on the bus: the +// readings disk-load keeps (its state `readings`, with the item's seven-day peak), the last good night per item restic +// keeps (its state `good`), and the manifests the indexer holds (for the bounds an item declares). Each kind, key and +// word is the controller's (mesh-controller cmd/mesh-controller/data.go), so the condition the keeper raises from it is +// the one the controller raised. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The states D13 reads (declared under `reads`). +const ( + ReadingsRead = health.ReadingsRead + GoodRead = health.GoodRead + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" +) + +// The kinds D13 raises. +const ( + kindDataShrank = "data-shrank" + kindEmptyReplacement = "empty-replacement" + kindDataHeldTwice = "data-held-twice" + kindDataQuiet = "data-quiet" + kindBackupStale = "backup-stale" + kindDataUnmeasured = "data-unmeasured" + kindDataMissing = "data-missing" + kindArrayDegraded = "array-degraded" + kindProtectionMissing = "protection-missing" + kindCleanupWaiting = "cleanup-waiting" + shrinkWindowDays = 7 + cleanupAfter = 30 * 24 * time.Hour + shrinkFloor int64 = 16 << 20 +) + +// dataFacts is everything D13 judges: the readings, the last good nights, each module's manifest, and the machines whose +// backup holder was heard from. +type dataFacts struct { + readings []health.Reading + good map[string]time.Time // by health.GoodKey + manifests map[string]catalogue.Manifest + // unmeasured are the machines that hold node-backup, are heard from, and have no reading of anything. + unmeasured []string + now time.Time +} + +// dataFacts reads them from the bus. +func (c *Checker) dataFacts() (dataFacts, error) { + f := dataFacts{good: map[string]time.Time{}, manifests: map[string]catalogue.Manifest{}, now: c.Now()} + keys, err := c.Store.Keys(ReadingsRead) + if err != nil { + return f, fmt.Errorf("disk-load's readings cannot be read: %w", err) + } + byNode := map[string]int{} + for _, k := range keys { + raw, ok, err := c.Store.Get(ReadingsRead, k) + if err != nil { + return f, fmt.Errorf("the reading %s cannot be read: %w", k, err) + } + if !ok { + continue + } + r, err := health.DecodeReading(raw) + if err != nil { + continue + } + f.readings = append(f.readings, r) + byNode[r.Node]++ + } + gkeys, err := c.Store.Keys(GoodRead) + if err != nil { + return f, fmt.Errorf("restic's last good nights cannot be read: %w", err) + } + for _, k := range gkeys { + raw, ok, err := c.Store.Get(GoodRead, k) + if err != nil { + return f, fmt.Errorf("the last good night %s cannot be read: %w", k, err) + } + if g, derr := health.DecodeGoodRow(raw); ok && derr == nil && !g.LastGood.IsZero() { + f.good[health.GoodKey(g.Node, g.Module, g.Item)] = g.LastGood + } + } + mods, err := c.Store.Keys(ModulesRead) + if err != nil { + return f, fmt.Errorf("the indexer's modules cannot be read: %w", err) + } + for _, name := range mods { + raw, ok, err := c.Store.Get(ModulesRead, name) + if err != nil || !ok { + continue + } + // Only the manifest's digest of the indexer's module, and the manifest's text: what the bounds are read from. + var m struct { + Manifest string `json:"manifest"` + } + if json.Unmarshal(raw, &m) != nil || m.Manifest == "" { + continue + } + mraw, ok, err := c.Store.Get(ManifestsRead, m.Manifest) + if err != nil || !ok { + continue + } + var doc struct { + JSON string `json:"manifest"` + } + if json.Unmarshal(mraw, &doc) != nil { + continue + } + var man catalogue.Manifest + if json.Unmarshal([]byte(doc.JSON), &man) == nil { + f.manifests[name] = man + } + } + heard, err := c.heard() + if err != nil { + return f, err + } + holders, err := c.holdersOf("node-backup") + if err != nil { + return f, err + } + for _, n := range holders { + if heard[n] && byNode[n] == 0 { + f.unmeasured = append(f.unmeasured, n) + } + } + return f, nil +} + +// probeData is D13. +func probeData(ctx context.Context, c *Checker) ([]Observation, error) { + f, err := c.dataFacts() + if err != nil { + return nil, err + } + return dataFindings(f), nil +} + +func severityOf(class string) health.Severity { + if class == catalogue.ClassIrreplaceable { + return Urgent + } + return Warning +} + +func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item } + +func comparable(precision string) bool { + return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ") +} + +func datasetOf(precision string) string { + rest, ok := strings.CutPrefix(precision, "dataset ") + if !ok { + return "" + } + name, _, _ := strings.Cut(rest, ":") + return name +} + +func orUnknownPath(p string) string { + if p == "" { + return "a path its backup holder never named" + } + return p +} + +func sizeWords(size *int64) string { + if size == nil || *size < 0 { + return "size unknown" + } + b := float64(*size) + for _, unit := range []string{"B", "KB", "MB", "GB"} { + if b < 1024 || unit == "GB" { + if unit == "B" { + return fmt.Sprintf("%d B", *size) + } + return fmt.Sprintf("%.1f %s", b, unit) + } + b /= 1024 + } + return "" +} + +// firstSeen is when an item was first measured on its node (the reading keeps it); for a reading written before it did, +// the oldest day its seven-day peak is made of, else its measurement: a lower bound on how long it has been declared. +func firstSeen(r health.Reading, now time.Time) time.Time { + if !r.FirstSeen.IsZero() { + return r.FirstSeen + } + if len(r.Days) > 0 { + if t, err := time.Parse("2006-01-02", r.Days[0].Day); err == nil { + return t + } + } + if r.MeasuredAt != nil { + return *r.MeasuredAt + } + return now +} + +func obs(kind string, id, machine string, sev health.Severity, summary, said string) Observation { + return Observation{Kind: kind, Token: kind, Severity: sev, Summary: summary, Said: said, + Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}} +} + +// dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was. +func dataFindings(f dataFacts) []Observation { + var out []Observation + arrays := map[string][]health.Reading{} + type shrunk struct { + machine, dataset, class string + size, peak int64 + items []string + } + shrunkDatasets := map[string]shrunk{} + byItem := map[string][]health.Reading{} + retiredCopies := map[string][]health.Reading{} + for _, m := range f.unmeasured { + o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+ + "declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data") + o.Confirm = true + out = append(out, o) + } + rs := append([]health.Reading(nil), f.readings...) + sort.Slice(rs, func(i, j int) bool { + return keyOfItem(rs[i].Node, rs[i].Module, rs[i].ID) < keyOfItem(rs[j].Node, rs[j].Module, rs[j].ID) + }) + for _, r := range rs { + class := r.Class + item, declared := f.manifests[r.Module].DataItem(r.ID) + id := keyOfItem(r.Node, r.Module, r.ID) + if !r.RetiredAt.IsZero() { + retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r) + if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) { + out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning, + fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+ + "`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again", + r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24), + r.RetiredWhy, r.Node, r.Module, r.ID, r.Module), "kept at "+orUnknownPath(r.Path))) + out[len(out)-1].Token = "cleanup" + } + continue + } + byItem[r.Module+"/"+r.ID] = append(byItem[r.Module+"/"+r.ID], r) + if !catalogue.Watched(class) { + continue + } + severity := severityOf(class) + if r.Redundancy != nil { + where := r.Node + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where + arrays[where] = append(arrays[where], r) + } else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.Error == "" { + out = append(out, obs(kindProtectionMissing, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+ + "and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see", + r.ID, r.Module, r.Node, class), + orUnknownPath(r.Path)+" is on no redundant storage the backup holder can read")) + } + peak := r.Peak + shrank := peak != nil && r.SizeBytes != nil && comparable(r.Precision) && *r.SizeBytes*2 < *peak && *peak-*r.SizeBytes >= shrinkFloor + switch { + case r.Error != "" && strings.Contains(r.Error, "does not exist"): + out = append(out, obs(kindDataMissing, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.ID, r.Module, r.Node, class), + orUnknownPath(r.Path)+" does not exist: "+firstLine(r.Error))) + case shrank && datasetOf(r.Precision) != "": + // Several items on one dataset share its size: one condition for the dataset, as loud as the most + // precious item on it. + k := r.Node + "/" + datasetOf(r.Precision) + ds := shrunkDatasets[k] + ds.machine, ds.dataset, ds.size, ds.peak = r.Node, datasetOf(r.Precision), *r.SizeBytes, *peak + ds.class = catalogue.StricterClass(ds.class, class) + ds.items = append(ds.items, r.Module+"/"+r.ID) + shrunkDatasets[k] = ds + case shrank: + out = append(out, obs(kindDataShrank, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+ + "held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+ + "good copy beside it", r.ID, r.Module, r.Node, class, sizeWords(r.SizeBytes), sizeWords(peak), shrinkWindowDays), "")) + } + if !declared { + continue + } + if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && f.now.Sub(*r.LastWrite) > within { + out = append(out, obs(kindDataQuiet, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+ + "whatever writes it has stopped", r.ID, r.Module, r.Node, r.LastWrite.UTC().Format(time.RFC3339), within), "")) + } + // A backup is required of what is irreplaceable and copied; of what is valuable it is the standard + // plan, said only where the machine was measured — where a holder is there to take it. + if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) { + within := item.BackupWithin() + last, has := f.good[health.GoodKey(r.Node, r.Module, r.ID)] + switch { + case !has && f.now.Sub(firstSeen(r, f.now)) > within: + out = append(out, obs(kindBackupStale, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+ + "declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)", + r.ID, r.Module, r.Node, class, f.now.Sub(firstSeen(r, f.now)).Round(time.Hour)), "")) + case has && f.now.Sub(last) > within: + out = append(out, obs(kindBackupStale, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+ + "of %s", r.ID, r.Module, r.Node, class, last.UTC().Format(time.RFC3339), within), "")) + } + } + } + keys := make([]string, 0, len(shrunkDatasets)) + for k := range shrunkDatasets { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + ds := shrunkDatasets[k] + out = append(out, obs(kindDataShrank, ds.machine+".dataset."+strings.ReplaceAll(ds.dataset, "/", "-"), ds.machine, + severityOf(ds.class), fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than "+ + "half of what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak), + shrinkWindowDays), "the dataset "+ds.dataset)) + } + // The redundant storage watched data is on: one condition per array, as loud as the most precious item on it — + // the array, not each item, is what degrades. + wheres := make([]string, 0, len(arrays)) + for w := range arrays { + wheres = append(wheres, w) + } + sort.Strings(wheres) + for _, where := range wheres { + items := arrays[where] + red := items[0].Redundancy + if red.Healthy != nil && *red.Healthy { + continue + } + class, machine := catalogue.ClassValuable, items[0].Node + var names []string + for _, r := range items { + class = catalogue.StricterClass(class, r.Class) + names = append(names, r.Module+"/"+r.ID) + } + state := "could not be read" + if red.Healthy != nil { + state = "is NOT healthy" + } + out = append(out, obs(kindArrayDegraded, machine+".array."+strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where), + machine, severityOf(class), fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, + strings.Join(names, ", "), state), fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said)))) + } + // The same item on several machines: a copy that is in use and far smaller than one kept retired elsewhere is an + // empty replacement. Only against a retired copy — a module running on two machines on purpose keeps two different + // sets of data. + items := make([]string, 0, len(byItem)) + for k := range byItem { + items = append(items, k) + } + sort.Strings(items) + for _, key := range items { + for _, a := range byItem[key] { + if a.SizeBytes == nil || !catalogue.Watched(a.Class) || !comparable(a.Precision) { + continue + } + for _, o := range retiredCopies[key] { + if o.Node == a.Node || o.SizeBytes == nil || !comparable(o.Precision) || !replacedByLess(*a.SizeBytes, *o.SizeBytes) { + continue + } + ob := obs(kindEmptyReplacement, keyOfItem(a.Node, a.Module, a.ID), a.Node, severityOf(a.Class), + fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ + "an empty replacement of its data. Move the data, or assign it back where its data is", + a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "") + ob.Subject.Also = []string{o.Node} + out = append(out, ob) + break + } + } + } + return out +} + +// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than half of it, and at +// least shrinkFloor less. +func replacedByLess(inUse, kept int64) bool { return inUse*2 < kept && kept-inUse >= shrinkFloor } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go new file mode 100644 index 00000000..a868ab33 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go @@ -0,0 +1,112 @@ +package main + +// D-fill (novox/hq ADR 0297 §6, issue 501): a module's bucket or log filling toward its cap is said before it is full. +// Every module's bucket (`KV__`) and log (`LOG__`) is read from the bus module's +// nats_streams, which answers each stream's size and its cap (max_bytes, mesh-catalog cab031bf). Said at three +// quarters of its cap, cleared below seven tenths: between the two a finding already written is kept, so one at its +// threshold is not raised and cleared on every run. The kind, key and words are the controller's +// (mesh-controller bucket_log_fill.go). + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The fill thresholds, in percent of a bucket's or log's cap. +const ( + fillRaiseAt = 75 + fillClearBelow = 70 +) + +// fillRow is what nats_streams answers of a stream's size and cap. +type fillRow struct { + Name string `json:"name"` + Bytes uint64 `json:"bytes"` + MaxBytes uint64 `json:"max_bytes"` +} + +// bucketOrLog is a module's bucket or log by its stream: the module, `bucket` or `log`, and its local name. False for +// a stream that is no module's bucket or log (the mesh's own streams). +func bucketOrLog(stream string) (module, kind, name string, ok bool) { + var rest string + switch { + case strings.HasPrefix(stream, "KV_"): + kind, rest = "bucket", strings.TrimPrefix(stream, "KV_") + case strings.HasPrefix(stream, buscompose.LogStreamPrefix): + kind, rest = "log", strings.TrimPrefix(stream, buscompose.LogStreamPrefix) + default: + return "", "", "", false + } + module, name, ok = strings.Cut(rest, "_") + if !ok || module == "" || name == "" { + return "", "", "", false + } + return module, kind, name, true +} + +// mibWords is a size as a person reads it, in MiB with one decimal. +func mibWords(b uint64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024)) } + +// fillKey is where a bucket's or log's fill finding is kept. +func fillKey(stream string) string { + return Observation{Subject: Subject{Scope: ScopeBus, ID: stream}, Token: "filling"}.Key() +} + +// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and, while its finding +// is written, at fillClearBelow or above too; nothing otherwise, which is what clears it. +func fillObservation(stream string, r fillRow, open bool) (Observation, bool) { + module, kind, name, ok := bucketOrLog(stream) + if !ok || r.MaxBytes == 0 { + return Observation{}, false // no module's, or with no cap: it cannot fill + } + // Compared in bytes, not rounded percent: 74.9% is not 75%. + atRaise := r.Bytes*100 >= r.MaxBytes*fillRaiseAt + aboveClear := r.Bytes*100 >= r.MaxBytes*fillClearBelow + if !atRaise && !(open && aboveClear) { + return Observation{}, false + } + pct := r.Bytes * 100 / r.MaxBytes + return Observation{Kind: "bucket-or-log-filling", Token: "filling", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: stream}, + Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write", + module, kind, name, mibWords(r.Bytes), mibWords(r.MaxBytes), pct), + Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes, + r.MaxBytes, pct), + Headline: "A module's bucket or log on the bus is filling up", + Needs: "decide whether to raise its cap or have the module keep less.", + Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it is " + + "full, the bus refuses what the module writes there.", + Resolved: "It has room again"}, true +} + +// probeFill is D-fill: every module's bucket and log on the bus judged against its cap. The bus module not answering +// is the probe failing to run, which clears nothing; a finding already written counts as open. +func probeFill(_ context.Context, c *Checker) ([]Observation, error) { + raw, err := c.Ask(StreamsTool, map[string]any{}) + if err != nil { + return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) + } + var a struct { + Detail []fillRow `json:"detail"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err) + } + written, err := c.written() + if err != nil { + return nil, fmt.Errorf("its own findings cannot be read, so a fill between the thresholds cannot be judged: %w", err) + } + sort.Slice(a.Detail, func(i, j int) bool { return a.Detail[i].Name < a.Detail[j].Name }) + var out []Observation + for _, r := range a.Detail { + if o, said := fillObservation(r.Name, r, written.has(fillKey(r.Name))); said { + out = append(out, o) + } + } + return sorted(out), nil +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go new file mode 100644 index 00000000..a25fab0a --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go @@ -0,0 +1,219 @@ +package main + +// The probes of the mesh's machines (novox/hq ADR 0564 §6, issue 1952): each reads its facts where they live on the +// bus — placement (the seat mesh-placement's state), who holds which seat (its seat-holders), whether a node is heard +// from (the watchman's presence) — and asks each node's seat holder what only that node knows. Its assertion, kind, +// key and words are the controller's (mesh-controller probes.go), so the condition the keeper raises from it is the one +// the controller raised. + +import ( + "context" + "encoding/json" + "fmt" + "net" + "regexp" + "slices" + "sort" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The states the machine probes read (declared under `reads`), and the verb D8 asks each node. +const ( + PlacementRead = delivery.AssignerModule + "." + "placement" + SeatHoldersRead = delivery.SeatHoldersRead + PresenceRead = delivery.WatchmanModule + ".presence" + BannedSeat = "node-intrusion-prevention" + BannedVerb = "seat:" + BannedSeat + ".banned" + ScopeMachine = health.ScopeMachine +) + +// placements is every node's placement, by node. +func (c *Checker) placements() (map[string]delivery.Placement, error) { + keys, err := c.Store.Keys(PlacementRead) + if err != nil { + return nil, fmt.Errorf("the placement cannot be read: %w", err) + } + out := map[string]delivery.Placement{} + for _, k := range keys { + raw, ok, err := c.Store.Get(PlacementRead, k) + if err != nil { + return nil, fmt.Errorf("the placement of %s cannot be read: %w", k, err) + } + if !ok { + continue + } + var p delivery.Placement + if err := json.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("the placement of %s is not readable: %w", k, err) + } + if p.Node == "" { + p.Node = k + } + out[p.Node] = p + } + return out, nil +} + +// heard is each node the watchman says is online. +func (c *Checker) heard() (map[string]bool, error) { + keys, err := c.Store.Keys(PresenceRead) + if err != nil { + return nil, fmt.Errorf("the watchman's presence cannot be read: %w", err) + } + out := map[string]bool{} + for _, k := range keys { + node, _, _, err := delivery.ParsePresenceKey(k) + if err != nil || node == "" { + continue + } + raw, ok, err := c.Store.Get(PresenceRead, k) + if err != nil { + return nil, fmt.Errorf("the presence of %s cannot be read: %w", node, err) + } + var p delivery.Presence + if ok && json.Unmarshal(raw, &p) == nil && p.Online { + out[node] = true + } + } + return out, nil +} + +// holdersOf is each node on which a module holds the node seat seat, sorted. +func (c *Checker) holdersOf(seat string) ([]string, error) { + keys, err := c.Store.Keys(SeatHoldersRead) + if err != nil { + return nil, fmt.Errorf("who holds which seat cannot be read: %w", err) + } + var out []string + for _, k := range keys { + if !strings.HasPrefix(k, seat+".") { + continue + } + raw, ok, err := c.Store.Get(SeatHoldersRead, k) + if err != nil { + return nil, fmt.Errorf("the holders of %s cannot be read: %w", k, err) + } + var sh delivery.SeatHolders + if !ok || json.Unmarshal(raw, &sh) != nil || sh.Seat != seat || sh.Node == "" || len(sh.Holders) == 0 { + continue + } + if !slices.Contains(out, sh.Node) { + out = append(out, sh.Node) + } + } + sort.Strings(out) + return out, nil +} + +// --- D8 ---------------------------------------------------------------------------------------------------------- + +// ipLike finds the addresses in a ban list, whatever shape its holder answers in. +var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`) + +// lookupHost resolves an endpoint's name; replaced in tests. +var lookupHost = func(ctx context.Context, host string) ([]string, error) { + return net.DefaultResolver.LookupHost(ctx, host) +} + +// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are every machine's +// private address and the address its endpoint names, from placement; the ban lists are asked of each machine's holder +// of node-intrusion-prevention that is heard from. +func probeBans(ctx context.Context, c *Checker) ([]Observation, error) { + places, err := c.placements() + if err != nil { + return nil, err + } + // address → whose, in machine names: what the summary says. The address, and the endpoint's name it was found + // by, are the evidence (ADR 0234 §6). + owned, via := map[string]string{}, map[string]string{} + nodes := make([]string, 0, len(places)) + for n := range places { + nodes = append(nodes, n) + } + sort.Strings(nodes) + for _, n := range nodes { + d := places[n].Decided + if a, _, _ := strings.Cut(d.Address, "/"); a != "" { + owned[a] = n + "'s private address" + } + if host, _, err := net.SplitHostPort(d.Endpoint); err == nil && host != "" { + if ip := net.ParseIP(host); ip != nil { + owned[ip.String()] = n + "'s endpoint" + } else if addrs, err := lookupHost(ctx, host); err == nil { + for _, a := range addrs { + owned[a], via[a] = n+"'s endpoint", host + } + } + } + } + heard, err := c.heard() + if err != nil { + return nil, err + } + all, err := c.holdersOf(BannedSeat) + if err != nil { + return nil, err + } + var holders []string + for _, n := range all { + if heard[n] { + holders = append(holders, n) + } + } + // Every machine asked at once: one after another, four holders that each wait their bound outlast the probe's + // thirty seconds, and the probe says nothing about any of them. + answers := make([]json.RawMessage, len(holders)) + errs := make([]error, len(holders)) + var wg sync.WaitGroup + for i, node := range holders { + wg.Add(1) + go func(i int, node string) { + defer wg.Done() + answers[i], errs[i] = c.Ask(BannedVerb+"@"+node, map[string]any{}) + }(i, node) + } + wg.Wait() + var out []Observation + var unasked []string + for i, node := range holders { + if errs[i] != nil { + unasked = append(unasked, errs[i].Error()) + continue + } + var banned, whose []string + for _, ip := range ipLike.FindAllString(string(answers[i]), -1) { + who, ours := owned[ip] + said := ip + " (" + who + ")" + if h := via[ip]; h != "" { + said = ip + " (" + who + ", " + h + ")" + } + if !ours || slices.Contains(banned, said) { + continue + } + banned = append(banned, said) + if !slices.Contains(whose, who) { + whose = append(whose, who) + } + } + if len(banned) > 0 { + out = append(out, Observation{Kind: "own-address-banned", Token: "bans-the-mesh", Severity: Urgent, + Subject: Subject{Scope: ScopeMachine, ID: node, Machine: node}, + Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es) — %s: the mesh is locked out "+ + "of itself there (ADR 0186)", node, len(banned), strings.Join(whose, ", ")), + Said: strings.Join(banned, ", "), + Headline: node + " has banned the mesh", + Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", node), + Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+ + "mesh is locked out of it.", node), + Resolved: node + " no longer bans the mesh"}) + } + } + if len(unasked) > 0 { + return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; ")) + } + return sorted(out), nil +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go new file mode 100644 index 00000000..2d46fba4 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go @@ -0,0 +1,254 @@ +package main + +// The account agents run as (novox/hq ADR 0266) and root where the trusted parties run (ADR 0259 §8), judged on the +// machine's own word: each node-engine says, in its health report (mesh-host.health, key ), its verdict on every +// account a module declares never to become root without a person. The judgement, its kind, key and words are the +// controller's (mesh-controller agent_account.go). Whether the verdict is current is the watchman's presence: the +// report is written on a change only, so a machine not heard from has no verdict on now. + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The engine's words and bounds the judgement reads (mesh-host rootsearch). +const ( + // HealthRead is the node-engines' health reports. + HealthRead = delivery.NodeHealthRead + // rootPending starts a verdict's reason while the engine's setuid search runs and no complete one judges. + rootPending = "not judged yet (search running)" + // rootQuiet is how long nothing is raised while an agent account waits for that search, counted from when this + // self-check first saw it waiting: past it, waiting is itself the urgent condition. It never makes it confined. + rootQuiet = 12 * time.Hour + stateHealthy = "healthy" + stateUnhealthy = "unhealthy" + stateUnknown = "unknown" +) + +// report is a node's health report, and whether it has one. +func (c *Checker) report(node string) (delivery.NodeHealth, bool, error) { + raw, ok, err := c.Store.Get(HealthRead, node) + if err != nil { + return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s cannot be read: %w", node, err) + } + if !ok { + return delivery.NodeHealth{}, false, nil + } + h, err := delivery.DecodeNodeHealth(raw) + if err != nil { + return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s: %w", node, err) + } + return h, true, nil +} + +// pendingSince is, by node, when this self-check first saw its agent account waiting for the setuid search: kept while +// it waits, forgotten once a verdict is complete. In this process alone: a restart of the node-engine does not restart +// it, and a restart of the self-check gives a waiting account at most one more quiet. +type pendingSince struct { + mu sync.Mutex + seen map[string]time.Time +} + +func (p *pendingSince) since(node string, now time.Time) time.Time { + p.mu.Lock() + defer p.mu.Unlock() + if p.seen == nil { + p.seen = map[string]time.Time{} + } + if at, ok := p.seen[node]; ok { + return at + } + p.seen[node] = now + return now +} + +func (p *pendingSince) judged(node string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.seen, node) +} + +// The kinds of an agent account's verdict, for the quiet a search earns. +const ( + verdictOther = iota // anything else: not heard, an older engine, no verdict, another unknown + verdictPending // waiting for the search, and otherwise healthy + verdictComplete // judged: healthy, or a way to root found +) + +// judgedConfined is the controller's judgement over one machine's report: confined only when the machine is heard +// from, its engine judges root, and every verdict on the agent account is healthy. why is said either way. +func judgedConfined(agent string, h delivery.NodeHealth, had, heard bool) (bool, string) { + switch { + case !had: + return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+ + "nothing of what it runs", agent) + case !heard: + return false, fmt.Sprintf("the agent account %s is not judged: the machine is not heard from, and a verdict "+ + "from before is not a verdict on now", agent) + case h.Contract < delivery.RootContract: + return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+ + "the judging of an account's root (its statement's contract is %d, the judging is %d)", + agent, h.Contract, delivery.RootContract) + } + verdicts := verdictsOn(agent, h) + if len(verdicts) == 0 { + return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+ + "verdict on it — no module there declares it never to become root, or the declaration naming it "+ + "has not been applied", agent) + } + for _, v := range verdicts { + switch v.State { + case stateHealthy: + case stateUnhealthy: + return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent, orNoneKnown(v.Reason), + v.Module, v.Resource) + default: + return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent, + orNoneKnown(v.Reason), v.Module, v.Resource, v.State) + } + } + return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent, + h.At.Local().Format("2006-01-02 15:04")) +} + +// verdictsOn is every verdict on the account, sorted by module and resource. +func verdictsOn(agent string, h delivery.NodeHealth) []delivery.RootVerdict { + var out []delivery.RootVerdict + for _, v := range h.Root { + if v.Account == agent { + out = append(out, v) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Resource < out[j].Module+out[j].Resource }) + return out +} + +// rootVerdictKind reads a report for the agent account: pending when every verdict on it is healthy or waits for the +// setuid search, at least one of them that; complete when every verdict is healthy or one found a way to root. +func rootVerdictKind(agent string, h delivery.NodeHealth, had, heard bool) int { + if !had || !heard || h.Contract < delivery.RootContract { + return verdictOther + } + pending, any := false, false + for _, v := range verdictsOn(agent, h) { + any = true + switch { + case v.State == stateHealthy: + case v.State == stateUnhealthy: + return verdictComplete + case v.State == stateUnknown && strings.HasPrefix(v.Reason, rootPending): + pending = true + default: + return verdictOther + } + } + switch { + case !any: + return verdictOther + case pending: + return verdictPending + } + return verdictComplete +} + +// agentQuiet says the one thing keeping a machine's agent account unjudged is the engine's setuid search, first seen +// waiting less than rootQuiet ago. +func (c *Checker) agentQuiet(node, agent string, h delivery.NodeHealth, had, heard bool) bool { + switch rootVerdictKind(agent, h, had, heard) { + case verdictComplete: + c.pending.judged(node) + case verdictPending: + now := c.clock() + return now.Sub(c.pending.since(node, now)) <= rootQuiet + } + return false +} + +// clock is the checker's clock, the wall clock where it has none. +func (c *Checker) clock() time.Time { + if c.Now != nil { + return c.Now() + } + return time.Now() +} + +// confinement is one machine's agent account judged: whether it names one, whether it is confined, why, and whether +// it is unjudged only because its search runs within the quiet. +type confinement struct { + Named, Confined, Quiet bool + Why string +} + +// confined judges node's agent account from its placement, its report and its presence. +func (c *Checker) confined(node string, place delivery.Placement, heard bool) (confinement, error) { + agent := place.Decided.AgentAccount + if agent == "" { + return confinement{Why: fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)", + node, orNoneKnown(place.Decided.Account))}, nil + } + h, had, err := c.report(node) + if err != nil { + return confinement{Named: true}, err + } + ok, why := judgedConfined(agent, h, had, heard) + return confinement{Named: true, Confined: ok, Why: why, Quiet: !ok && c.agentQuiet(node, agent, h, had, heard)}, nil +} + +// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest +// statement, unable to become root without a person (ADR 0266). +func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) { + places, err := c.placements() + if err != nil { + return nil, err + } + heard, err := c.heard() + if err != nil { + return nil, err + } + nodes := make([]string, 0, len(places)) + for n := range places { + nodes = append(nodes, n) + } + sort.Strings(nodes) + var out []Observation + for _, n := range nodes { + if places[n].Decided.AgentAccount == "" { + continue + } + v, err := c.confined(n, places[n], heard[n]) + if err != nil { + return nil, err + } + // Not judged yet only because the engine's search runs: not the urgent condition after every restart. Loud + // again once the search fails, waits past the quiet, or the machine is not heard from. + if v.Confined || v.Quiet { + continue + } + out = append(out, Observation{Kind: "agent-can-become-root", Token: "agent-root", Severity: Urgent, + Subject: Subject{Scope: ScopeMachine, ID: n, Machine: n}, + Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+ + "answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why), + Said: v.Why, + Headline: "Sessions on " + n + " could become root", + Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.", + Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+ + "can take over the machine without you. The machine cannot show that this holds now, so an answer "+ + "from your phone authorises nothing there until it does.", n), + Resolved: "Sessions on " + n + " cannot become root again"}) + } + return sorted(out), nil +} + +// orNoneKnown is a value, or that none is known. +func orNoneKnown(s string) string { + if strings.TrimSpace(s) == "" { + return "none known" + } + return s +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go new file mode 100644 index 00000000..03eb637a --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -0,0 +1,455 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed. +type bus struct { + drop map[string]bool // stream or stream.consumer missing + change func(rows map[string]*streamRow) + closed string + signals error + asksFail error + // banned is each node's answer to node-intrusion-prevention.banned; a node absent does not answer. + banned map[string]string + // fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap. + fills []fillRow +} + +func (b bus) rows() map[string]*streamRow { + rows := map[string]*streamRow{} + for _, s := range buscompose.MeshStreams() { + if b.drop[s.Name] { + continue + } + retention := "limits" + per := int64(s.MaxMsgsPerSubject) + switch s.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + per = 1 + } + rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per} + } + for _, c := range buscompose.MeshConsumers() { + r := rows[c.Stream] + if r == nil || b.drop[c.Stream+"."+c.Name] { + continue + } + kind := "pull" + if c.Push || c.Queue != "" { + kind = "push" + } + r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)}) + } + if b.change != nil { + b.change(rows) + } + return rows +} + +func (b bus) ask(key string, body any) (json.RawMessage, error) { + if b.asksFail != nil { + return nil, b.asksFail + } + switch key { + case StreamsTool: + stream, _ := body.(map[string]any)["stream"].(string) + var detail []streamRow + for name, r := range b.rows() { + if stream == "" { + row := *r + row.Consumers = nil + detail = append(detail, row) + } else if name == stream { + detail = append(detail, *r) + } + } + if stream == "" && len(b.fills) > 0 { + var rows []any + for _, r := range detail { + rows = append(rows, r) + } + for _, f := range b.fills { + rows = append(rows, f) + } + return json.Marshal(map[string]any{"detail": rows}) + } + if stream != "" && len(detail) == 0 { + return nil, errors.New("no stream called " + stream) + } + return json.Marshal(map[string]any{"detail": detail}) + case ClosedTool: + if b.closed == "" { + return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil + } + return json.RawMessage(b.closed), nil + case SignalsVerb: + if b.signals != nil { + return nil, b.signals + } + return json.RawMessage(`{"signals":[]}`), nil + } + if node, ok := strings.CutPrefix(key, BannedVerb+"@"); ok { + if a, ok := b.banned[node]; ok { + return json.RawMessage(a), nil + } + } + return nil, errors.New("nats: no responders available for request") +} + +func keysOf(found []Observation) []string { + var out []string + for _, f := range found { + out = append(out, f.Key()) + } + return out +} + +// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2). +func TestTheBusProbes(t *testing.T) { + cases := []struct { + name string + probe func(context.Context, *Checker) ([]Observation, error) + bus bus + want []string + err string + mesh *memStore // the bus states the machine probes read; nil reads none + }{ + {"D6 all as defined", probeConsumers, bus{}, nil, "", nil}, + {"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}}, + []string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, "", nil}, + {"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}}, + []string{"bus.CONTROL.controller.missing"}, "", nil}, + {"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) { + for i, c := range rows["EVENTS"].Consumers { + if c.Name == "controller" { + rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000 + } + } + }}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, "", nil}, + {"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil}, + {"D7 all as defined", probeStreams, bus{}, nil, "", nil}, + {"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) { + rows["EVENTS"].Retention = "workqueue" + }}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, "", nil}, + {"D15 quiet", probeReconnects, bus{}, nil, "", nil}, + {"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[ + {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, + {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, + []string{"bus.g14.node-tools.bus-reconnects"}, "", nil}, + {"DW answers", probeWatchdogs, bus{}, nil, "", nil}, + {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, "", nil}, + {"H-bus healthy", probeBusHealth, bus{}, nil, "", nil}, + {"H-bus a stream and a consumer missing", probeBusHealth, + bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, "", nil}, + {"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed", nil}, + // D8 (issue 238): placement's addresses and endpoints against each heard holder's ban list. + {"D8 nothing of the mesh banned", probeBans, bus{banned: map[string]string{"ace": `{"banned":["198.51.100.7"]}`, + "novox": `{"banned":[]}`}}, nil, "", machines()}, + {"D8 a private address and an endpoint banned", probeBans, bus{banned: map[string]string{ + "ace": `{"jails":{"sshd":["10.10.0.1","198.51.100.7"]}}`, "novox": `{"banned":["203.0.113.5"]}`}}, + []string{"machine.ace.bans-the-mesh", "machine.novox.bans-the-mesh"}, "", machines()}, + {"D8 a machine not heard from is not asked", probeBans, bus{banned: map[string]string{"novox": `{"banned":[]}`, + "ace": `{"banned":[]}`}}, nil, "", machines()}, + {"D8 a heard holder that does not answer is no pass", probeBans, bus{banned: map[string]string{}}, nil, + "a ban list could not be read", machines()}, + // D13 (issue 273, ADR 0233): the readings disk-load keeps and the last good nights restic keeps, against the bounds + // the manifests declare; the kinds, keys and words are the controller's. + {"D13 all held", probeData, bus{}, nil, "", dataMesh(func(*dataRig) {})}, + {"D13 shrank, gone, quiet, unprotected and stale", probeData, bus{}, []string{ + "machine.ace.pg.data.data-quiet", "machine.ace.pg.data.backup-stale", "machine.ace.pg.dumps.data-missing", + "machine.ace.plex.movies.protection-missing", "machine.ace.plex.thumbs.data-shrank", + "machine.ace.array.md-md127.array-degraded"}, + "", dataMesh(func(r *dataRig) { + r.size("ace", "pg", "data", 100<<20, 100<<20).write("ace", "pg", "data", -80*time.Hour) + r.gone("ace", "pg", "dumps") + r.size("ace", "plex", "thumbs", 10<<20, 600<<20) + r.size("ace", "plex", "movies", 1<<30, 1<<30).noArray("ace", "plex", "movies") + r.size("ace", "plex", "library", 1<<30, 1<<30).array("ace", "plex", "library", "md", "md127", false) + r.lastGood("ace", "pg", "data", -100*time.Hour) + })}, + {"D13 a dataset shrank once for its items", probeData, bus{}, []string{"machine.ace.dataset.storage-media.data-shrank"}, + "", dataMesh(func(r *dataRig) { + r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media") + r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media") + })}, + {"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{ + "machine.ace.pg.dumps.cleanup"}, "", dataMesh(func(r *dataRig) { + r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour) + r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour) + })}, + {"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{}, + []string{"machine.ace.plex.thumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) { + r.size("ace", "plex", "thumbs", 10<<20, 10<<20) + r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour) + })}, + {"D13 no good backup is counted from when the item was first measured", probeData, bus{}, + []string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) { + delete(r.good, health.ReadingKey("ace", "pg", "data")) + delete(r.good, health.ReadingKey("ace", "plex", "meta")) + r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour) + })}, + {"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "", + dataMesh(func(r *dataRig) { r.holder("g14") })}, + // D-fill (ADR 0297 §6): a module's bucket or log at three quarters of its cap, kept until under seven tenths. + {"D-fill all with room", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 74, MaxBytes: 100}, + {Name: "LOG_mesh-delivery-builder_build-log", Bytes: 10, MaxBytes: 100}, {Name: "KV_nocap_x", Bytes: 999}}}, nil, "", nil}, + {"D-fill a bucket and a log filling", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 75, MaxBytes: 100}, + {Name: "LOG_mesh-delivery-builder_build-log", Bytes: 99, MaxBytes: 100}}}, + []string{"bus.KV_mesh-issues_issues.filling", "bus.LOG_mesh-delivery-builder_build-log.filling"}, "", nil}, + {"D-fill between the thresholds raises nothing new", probeFill, + bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, nil, "", nil}, + {"D-fill between the thresholds keeps what is said", probeFill, + bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, + []string{"bus.KV_mesh-issues_issues.filling"}, "", filling("KV_mesh-issues_issues")}, + {"D-fill the bus module silent", probeFill, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil}, + // DA (ADR 0266): the agent account each machine names, judged on its engine's report while it is heard from. + {"DA confined", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": stateHealthy})}, + {"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + agents(map[string]string{"novox": stateUnhealthy})}, + {"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + agents(map[string]string{"novox": ""})}, + {"DA the search runs, within its quiet", probeAgentAccounts, bus{}, nil, "", + agents(map[string]string{"novox": "pending"})}, + {"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-root"}, + "", agents(map[string]string{"novox": stateHealthy, "g14": stateHealthy})}, + {"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + agents(map[string]string{"novox": "old"})}, + } + for _, tc := range cases { + c := &Checker{Ask: tc.bus.ask, Store: tc.mesh, Now: func() time.Time { return dataNow }} + if tc.mesh == nil { + c.Store = newStore() + } + found, err := tc.probe(context.Background(), c) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Errorf("%s: error %v, want %q", tc.name, err, tc.err) + } + continue + } + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if got := keysOf(found); !slices.Equal(got, tc.want) { + t.Errorf("%s: found %v, want %v", tc.name, got, tc.want) + } + } +} + +// machines is a mesh of three: novox the hub with its endpoint, ace and g14 on the private network; ace and novox +// hold node-intrusion-prevention and are heard from, g14 holds it and is not. +func machines() *memStore { + s := newStore() + for _, p := range []struct { + node, address, endpoint string + online bool + }{{"novox", "10.10.0.1/16", "203.0.113.5:51820", true}, {"ace", "10.10.0.2/16", "", true}, {"g14", "10.10.0.3/16", "", false}} { + _ = s.Put(PlacementRead, p.node, delivery.Placement{V: 1, Node: p.node, + Decided: delivery.NodeDecided{Address: p.address, Endpoint: p.endpoint, Hub: p.endpoint != ""}}) + _ = s.Put(PresenceRead, p.node, delivery.Presence{V: 1, Online: p.online}) + key, _ := delivery.SeatHoldersKey(BannedSeat, p.node) + _ = s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: BannedSeat, Scope: "node", Node: p.node, + Holders: []delivery.SeatHolder{{Module: "fail2ban", Node: p.node}}}) + } + return s +} + +// filling is a store holding the self-check's written fill finding of a stream: the condition open. +func filling(stream string) *memStore { + s := newStore() + o := Observation{Kind: "bucket-or-log-filling", Token: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream}, + Summary: "filling", Source: "D-fill"} + raw, _ := health.EncodeFinding(o.Finding("run", time.Now())) + _ = s.Put(FindingsState, o.Key(), json.RawMessage(raw)) + return s +} + +// agents is machines() where each node named names the agent account "agent" and its report holds the verdict on it: +// a state, "" for none, "pending" for one waiting for its search, "old" for an engine older than the judging. +func agents(verdicts map[string]string) *memStore { + s := machines() + for node, v := range verdicts { + raw, _, _ := s.Get(PlacementRead, node) + var p delivery.Placement + _ = json.Unmarshal(raw, &p) + p.Decided.AgentAccount = "agent" + _ = s.Put(PlacementRead, node, p) + h := delivery.NodeHealth{V: 1, Node: node, Contract: 13} + switch v { + case "": + case "old": + h.Contract = 2 + h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: stateHealthy}} + case "pending": + h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", + State: stateUnknown, Reason: rootPending + ": 3 of 9 trees"}} + default: + h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: v, + Reason: "can become root without a person: a sudo rule"}} + } + raw, _ = delivery.EncodeNodeHealth(h) + _ = s.Put(HealthRead, node, json.RawMessage(raw)) + } + return s +} + +// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what +// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). +func TestTheRegistryIsADR0564sTable(t *testing.T) { + moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", + "DW", "H-bus"} + var ids []string + for _, p := range Registry() { + ids = append(ids, p.ID) + if p.Kind == "" || p.Asserts == "" { + t.Errorf("%s has no kind or assertion", p.ID) + } + if (p.Run == nil) == (p.Deferred == "") { + t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID) + } + } + slices.Sort(ids) + slices.Sort(moved) + if !slices.Equal(ids, moved) { + t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved) + } +} + +// ---- D13's rig ------------------------------------------------------------------------------------------------ + +var dataNow = time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + +// dataRig is the mesh D13 reads: a manifest per module with the items it declares, a reading per item, a last good night. +type dataRig struct { + s *memStore + readings map[string]*health.Reading + good map[string]time.Time +} + +func (r *dataRig) key(node, module, item string) string { return health.ReadingKey(node, module, item) } + +func (r *dataRig) reading(node, module, item string) *health.Reading { + k := r.key(node, module, item) + if r.readings[k] == nil { + at := dataNow + r.readings[k] = &health.Reading{V: health.ReadingVersion, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module, + ID: item, Class: catalogue.ClassValuable, Path: "/srv/" + module + "/" + item}, Measured: health.Measured{MeasuredAt: &at, Precision: "exact"}} + } + return r.readings[k] +} + +func (r *dataRig) size(node, module, item string, size, peak int64) *dataRig { + x := r.reading(node, module, item) + x.SizeBytes, x.Peak = &size, &peak + return r +} + +func (r *dataRig) write(node, module, item string, ago time.Duration) *dataRig { + t := dataNow.Add(ago) + r.reading(node, module, item).LastWrite = &t + return r +} + +func (r *dataRig) gone(node, module, item string) *dataRig { + r.reading(node, module, item).Error = "/srv/" + module + "/" + item + " does not exist" + return r +} + +func (r *dataRig) dataset(node, module, item, name string) *dataRig { + r.reading(node, module, item).Precision = "dataset " + name + ": its whole size" + return r +} + +func (r *dataRig) array(node, module, item, kind, where string, healthy bool) *dataRig { + r.reading(node, module, item).Redundancy = &health.Redundancy{Kind: kind, Where: where, Healthy: &healthy, Said: "a member is missing"} + return r +} + +func (r *dataRig) noArray(node, module, item string) *dataRig { + r.reading(node, module, item).Redundancy = nil + return r +} + +func (r *dataRig) lastGood(node, module, item string, ago time.Duration) *dataRig { + r.good[r.key(node, module, item)] = dataNow.Add(ago) + return r +} + +func (r *dataRig) retired(node, module, item string, ago time.Duration) *dataRig { + x := r.reading(node, module, item) + x.RetiredAt, x.RetiredWhy = dataNow.Add(ago), "no longer declared on "+node + return r +} + +func (r *dataRig) firstSeen(node, module, item string, ago time.Duration) *dataRig { + r.reading(node, module, item).FirstSeen = dataNow.Add(ago) + return r +} + +func (r *dataRig) holder(node string) { + key, _ := delivery.SeatHoldersKey("node-backup", node) + _ = r.s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: "node-backup", Scope: "node", Node: node, + Holders: []delivery.SeatHolder{{Module: "restic", Node: node}}}) + _ = r.s.Put(PresenceRead, node, delivery.Presence{V: 1, Online: true}) +} + +// dataMesh builds the mesh: pg declares data (backed up within 48 h, written all the time within 24 h) and dumps; plex +// declares movies (protected by redundancy), thumbs, meta and library (all valuable); every item is read as held, and what +// is given changes it. +func dataMesh(change func(*dataRig)) *memStore { + r := &dataRig{s: newStore(), readings: map[string]*health.Reading{}, good: map[string]time.Time{}} + r.holder("ace") + declare := func(module string, items map[string]catalogue.DataItem) { + var man catalogue.Manifest + man.Module = module + man.Data = &catalogue.Data{} + for id, it := range items { + it.ID = id + man.Data.Own = append(man.Data.Own, it) + } + raw, _ := json.Marshal(man) + digest := delivery.ManifestDigest(raw) + _ = r.s.Put(ManifestsRead, digest, delivery.Manifest{V: 1, Module: module, JSON: string(raw)}) + _ = r.s.Put(ModulesRead, module, delivery.Module{V: 1, Manifest: digest}) + } + declare("pg", map[string]catalogue.DataItem{ + "data": {Class: catalogue.ClassValuable, Within: "48h", Active: "24h"}, + "dumps": {Class: catalogue.ClassValuable}}) + declare("plex", map[string]catalogue.DataItem{ + "movies": {Class: catalogue.ClassValuable, Redundancy: "the array"}, + "thumbs": {Class: catalogue.ClassValuable}, + "meta": {Class: catalogue.ClassValuable}, + "library": {Class: catalogue.ClassValuable, Redundancy: "the array"}}) + for _, i := range [][2]string{{"pg", "data"}, {"pg", "dumps"}, {"plex", "movies"}, {"plex", "thumbs"}, {"plex", "meta"}, {"plex", "library"}} { + r.size("ace", i[0], i[1], 1<<30, 1<<30).lastGood("ace", i[0], i[1], -time.Hour).write("ace", i[0], i[1], -time.Minute) + if i[0] == "plex" && (i[1] == "movies" || i[1] == "library") { + r.array("ace", i[0], i[1], "md", "md127", true) + } + } + change(r) + for k, x := range r.readings { + _ = r.s.Put(ReadingsRead, k, x) + } + for k, t := range r.good { + node, rest, _ := strings.Cut(k, ".") + module, item, _ := strings.Cut(rest, ".") + _ = r.s.Put(GoodRead, k, health.GoodRow{V: health.GoodVersion, Node: node, Module: module, Item: item, CoveredBy: "/x", LastGood: t}) + } + return r.s +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/reads-outside b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside new file mode 100644 index 00000000..cb8f579b --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-self-check/module.json diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go new file mode 100644 index 00000000..174ff884 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -0,0 +1,417 @@ +package main + +// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five +// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket +// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the +// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the +// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is +// said in the verdict at once and written as a finding of its own when the next run cannot run it either. + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-self-check" + Seat = "mesh-self-check" +) + +// The states it keeps, and the event that ends every run. +const ( + FindingsState = health.FindingsState + VerdictState = "verdict" + VerdictsState = "verdicts" + LastKey = "last" + CheckedEvent = health.EventChecked + // CheckedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent +) + +// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). +var ( + Every = 5 * time.Minute + FirstAfter = time.Minute + Within = 30 * time.Second +) + +// The verdicts a probe can have, as mesh-sdk's health names them. +const ( + VerdictPass = health.VerdictPassed + VerdictFail = health.VerdictFailed + VerdictFailedToRun = health.VerdictFailedToRun + VerdictDeferred = health.VerdictDeferred +) + +// sourceSelf is what writes a probe's failure to run: the self-check itself. +const sourceSelf = "doctor" + +// Store is the bus's buckets as this module reaches them, through its runtime. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// On says whether the setting lists the probe. +func (c *Checker) On(id string) bool { + if c.Enabled == nil { + return true + } + return slices.Contains(c.Enabled(), id) +} + +// DefaultProbes are the probes that run unless the setting says otherwise. +var DefaultProbes = []string{"D13", "D-fill"} + +// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits. +type Probe struct { + ID string + Asserts string + From string + Kind string + Raises []string + // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): + // answered deferred, never a pass. + Deferred string + Run func(ctx context.Context, c *Checker) ([]Observation, error) +} + +// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's +// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it. +type ( + ProbeVerdict = health.ProbeVerdict + Counts = health.CheckedCounts + Run = health.Checked +) + +// Checker is the self-check: the registry and what its probes reach. +type Checker struct { + Store Store + // Enabled is the setting `probes`: the ids of the probes that run. Nil runs every probe in the registry; one not + // listed runs nothing and raises nothing (the operator, 2026-10-30). + Enabled func() []string + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) + Host string + Registry []Probe + + running sync.Mutex + mu sync.Mutex + n uint64 + last *Run + // seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is + // written only when two runs in a row see it, or while it is already written (novox/hq issue 277). + seen map[string]bool + // unsaid counts the runs whose `checked` could not be said since the start. + unsaid int + // pending is when each machine's agent account was first seen waiting for its setuid search (DA, D-root). + pending pendingSince +} + +// LastRun is the last run, nil before the first. +func (c *Checker) LastRun() *Run { + c.mu.Lock() + defer c.mu.Unlock() + return c.last +} + +type result struct { + found []Observation + err error + took time.Duration +} + +// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`. +// One run at a time: a person's run during a scheduled one waits for it. +func (c *Checker) RunOnce(ctx context.Context, why string) Run { + c.running.Lock() + defer c.running.Unlock() + c.mu.Lock() + c.n++ + n := c.n + c.mu.Unlock() + started := c.Now() + run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(), + IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why} + + results := make([]result, len(c.Registry)) + var wg sync.WaitGroup + for i, p := range c.Registry { + if p.Run == nil || !c.On(p.ID) { + continue + } + wg.Add(1) + go func(i int, p Probe) { + defer wg.Done() + results[i] = c.probe(ctx, p) + }(i, p) + } + wg.Wait() + + written, readErr := c.written() + seen := map[string]bool{} + want := map[string]Observation{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Observation + for i, p := range c.Registry { + if !c.On(p.ID) { + // Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no + // verdict in the run. + ran[p.ID] = true + continue + } + v := ProbeVerdict{ID: p.ID} + r := results[i] + switch { + case p.Run == nil: + v.Verdict = VerdictDeferred + v.Error = "deferred: " + p.Deferred + run.Counts.Deferred++ + case r.err != nil: + v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() + run.Counts.FailedToRun++ + blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, + Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), + Said: firstLine(r.err.Error())}) + default: + ran[p.ID] = true + for _, f := range r.found { + if f.Kind == "" { + f.Kind = p.Kind + } + f.Source = p.ID + f = worded(f) + key := f.Key() + seen[key] = true + if f.Confirm && !c.seenBefore(key) && !written.has(key) { + v.Unconfirmed = append(v.Unconfirmed, f.Summary) + continue + } + want[key] = f + v.Found = append(v.Found, f.Summary) + } + if len(v.Found) == 0 { + v.Verdict = VerdictPass + run.Counts.Passed++ + } else { + v.Verdict = VerdictFail + run.Counts.Failed++ + } + } + if p.Run != nil { + v.Took = r.took.Round(time.Millisecond).String() + } + run.Probes = append(run.Probes, v) + } + ran[sourceSelf] = true + for _, f := range blind { + f = worded(f) + key := f.Key() + seen[key] = true + if c.seenBefore(key) || written.has(key) { + want[key] = f + } + } + if readErr != nil { + // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. + c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) + } else if err := c.write(written, want, ran, run.Run); err != nil { + c.Logf("[%s] what this run found is not all written: %v", Module, err) + } + c.mu.Lock() + c.seen = seen + c.mu.Unlock() + + run.At = c.Now().UTC() + run.Took = run.At.Sub(started).Round(time.Millisecond).String() + c.mu.Lock() + run.Unsaid = c.unsaid + c.mu.Unlock() + c.keep(run) + return run +} + +// probe runs one probe within its bound; a panic or an overrun is an error, never a pass. +func (c *Checker) probe(ctx context.Context, p Probe) result { + probing, cancel := context.WithTimeout(ctx, Within) + defer cancel() + began := c.Now() + done := make(chan result, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- result{err: fmt.Errorf("the probe panicked: %v", r)} + } + }() + found, err := p.Run(probing, c) + done <- result{found: found, err: err} + }() + select { + case r := <-done: + r.took = c.Now().Sub(began) + return r + case <-probing.Done(): + return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)} + } +} + +func (c *Checker) seenBefore(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.seen[key] +} + +// writtenFindings are the findings in the bucket, by key. +type writtenFindings map[string]health.Finding + +func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } + +// written reads every finding this module has written. +func (c *Checker) written() (writtenFindings, error) { + keys, err := c.Store.Keys(FindingsState) + if err != nil { + return writtenFindings{}, err + } + out := writtenFindings{} + for _, k := range keys { + raw, found, err := c.Store.Get(FindingsState, k) + if err != nil { + return out, err + } + if !found { + continue + } + f, err := health.DecodeFinding(raw) + if err != nil { + return out, fmt.Errorf("the finding %s is not readable: %w", k, err) + } + out[k] = f + } + return out, nil +} + +// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding +// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). +func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error { + var errs []string + keys := make([]string, 0, len(want)) + for k := range want { + keys = append(keys, k) + } + sort.Strings(keys) + now := c.Now() + for _, k := range keys { + f := want[k].Finding(run, now) + if was, ok := written[k]; ok && says(was) == says(f) { + continue + } + raw, err := health.EncodeFinding(f) + if err == nil { + err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + for k, f := range written { + if _, still := want[k]; still || !ran[f.Source] { + continue + } + if err := c.Store.Delete(FindingsState, k); err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} + +// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in +// the log: the watchman's S10 says the self-check silent when `checked` stops. +func (c *Checker) keep(run Run) { + c.mu.Lock() + c.last = &run + c.mu.Unlock() + if err := c.Store.Put(VerdictState, LastKey, run); err != nil { + c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err) + } + if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil { + c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err) + } + if c.Publish == nil { + return + } + if err := c.Publish(CheckedSubject, run, run.Run); err != nil { + c.mu.Lock() + c.unsaid++ + c.mu.Unlock() + c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v", + Module, run.Run, err) + } +} + +// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314). +const overviewFindings = 20 + +// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all. +func Overview(run Run) Run { + probes := make([]ProbeVerdict, len(run.Probes)) + for i, p := range run.Probes { + total := len(p.Found) + len(p.Unconfirmed) + if total > overviewFindings { + p.FoundCount = len(p.Found) + if len(p.Found) > overviewFindings { + p.Found = p.Found[:overviewFindings:overviewFindings] + } + if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room { + p.Unconfirmed = p.Unconfirmed[:room:room] + } + p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all", + p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID) + } + probes[i] = p + } + run.Probes = probes + return run +} + +// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every. +func (c *Checker) Keep(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-time.After(FirstAfter): + } + tick := time.NewTicker(Every) + defer tick.Stop() + for { + c.RunOnce(ctx, "the schedule") + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go new file mode 100644 index 00000000..50557340 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -0,0 +1,295 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// memStore is the bus's buckets in memory, counting the writes. +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + puts map[string]int + keysErr error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) Put(state, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + s.kv[state][key] = raw + s.puts[state+"/"+key]++ + return nil +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.keysErr != nil && state == FindingsState { + return nil, s.keysErr + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) findingKeys() []string { + keys, _ := s.Keys(FindingsState) + return keys +} + +// checker is a self-check over memory, with a registry the test drives. +func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { + var said []string + now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) + c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Publish: func(subject string, body any, id string) error { + raw, _ := json.Marshal(body) + said = append(said, subject+" "+id+" "+string(raw)) + return nil + }} + return c, &said +} + +// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that +// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can +// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). +func TestARunWritesFindingsAsTheyChange(t *testing.T) { + var found []Observation + var fail error + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }} + deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} + missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + Summary: "the stream EVENTS is not on the bus"} + behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} + store := newStore() + c, said := checker(store, steady, deferred) + + steps := []struct { + name string + found []Observation + fail error + want []string // finding keys written after the run + verdict string + missingPs int // puts of the missing finding so far + }{ + {"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []Observation{missing, behind}, nil, + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"blind once: clears nothing, held", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, + {"blind twice: its own finding", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1}, + {"no longer seen: cleared", nil, nil, nil, VerdictPass, 1}, + } + for _, s := range steps { + found, fail = s.found, s.fail + run := c.RunOnce(context.Background(), "the test") + if got := store.findingKeys(); !slices.Equal(got, s.want) { + t.Errorf("%s: findings %v, want %v", s.name, got, s.want) + } + if run.Probes[0].Verdict != s.verdict { + t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict) + } + if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") { + t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1]) + } + if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs { + t.Errorf("%s: the unchanged finding was written %d times", s.name, n) + } + } + raw, _, _ := store.Get(FindingsState, "probe.P1.failed") + if raw != nil { + t.Errorf("the blind finding outlived the probe running again: %s", raw) + } + // Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body. + if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) { + t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps)) + } + // Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id. + if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") { + t.Fatalf("said %v", *said) + } + _, rest, _ := strings.Cut((*said)[0], " ") + _, rest, _ = strings.Cut(rest, " ") + var body map[string]any + _ = json.Unmarshal([]byte(rest), &body) + for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} { + if _, ok := body[field]; !ok { + t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body) + } + } + last, _, _ := store.Get(VerdictState, LastKey) + var r Run + _ = json.Unmarshal(last, &r) + if r.Counts.Passed != 1 || r.Counts.Deferred != 1 { + t.Errorf("the last verdict counts %+v", r.Counts) + } + // A finding carries the operator's words of its kind, the probe that found it and who observed it. + found, fail = []Observation{missing}, nil + c.RunOnce(context.Background(), "the test") + raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") + f, err := health.DecodeFinding(raw) + if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || + f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { + t.Errorf("the written finding is %+v", f) + } + // Blind to its own bucket, a run writes and clears nothing. + store.keysErr = errors.New("the bus is away") + found = nil + c.RunOnce(context.Background(), "the test") + store.keysErr = nil + if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) { + t.Errorf("a run blind to its findings changed them: %v", got) + } +} + +// A probe that panics or overruns its bound could not run: never a pass. +func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { + within := Within + Within = 50 * time.Millisecond + t.Cleanup(func() { Within = within }) + for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){ + "panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil }, + } { + c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) + c.Now = time.Now + r := c.RunOnce(context.Background(), "the test") + if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 { + t.Errorf("%s: %+v", name, r.Probes[0]) + } + } +} + +// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer +// every state the module keeps (ADR 0564 §12). +func TestTheVerbsAndReads(t *testing.T) { + store := newStore() + many := make([]Observation, 25) + for i := range many { + many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + } + c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", + Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }}) + if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { + t.Fatalf("before a run: %v", err) + } + if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil { + t.Fatal(err) + } + over, _ := c.Doctor(false, "", "") + if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" { + t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More) + } + whole, _ := c.Doctor(false, "d7", "") + if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 { + t.Errorf("one probe whole carries %d findings", len(v.Found)) + } + if _, err := c.Doctor(false, "D99", ""); err == nil { + t.Error("an unknown probe answered") + } + // A fresh checker over the same bucket answers the last verdict from it. + fresh, _ := checker(store) + if _, err := fresh.Doctor(false, "", ""); err != nil { + t.Errorf("the last verdict is not read from its bucket: %v", err) + } + fs, err := c.Findings("") + if err != nil || fs.(map[string]any)["count"] != 25 { + t.Errorf("findings answered %v, %v", fs, err) + } + if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") { + t.Errorf("one finding answered %v, %v", one, err) + } + vs, err := c.Verdicts("", 0) + if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 { + t.Errorf("verdicts answered %v, %v", vs, err) + } + if _, err := c.Verdicts("doctor-1-1", 0); err == nil { + t.Error("an unknown run answered") + } + if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" { + t.Errorf("probes answered %v", p) + } +} + +// Only the probes the setting `probes` lists run: one not listed runs nothing, has no verdict, clears what it raised before, +// and is reported by the verb as off by setting; the default list is D13 and D-fill (the operator, 2026-10-30). +func TestOnlyTheProbesTheSettingListsRun(t *testing.T) { + ran := map[string]int{} + finds := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "gone"} + probe := func(id string) Probe { + return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { + ran[id]++ + return []Observation{finds}, nil + }} + } + store := newStore() + c, _ := checker(store, probe("D7"), probe("D13")) + c.Enabled = func() []string { return []string{"D7", "D13"} } + run := c.RunOnce(context.Background(), "test") + if ran["D7"] != 1 || ran["D13"] != 1 || len(run.Probes) != 2 || len(store.findingKeys()) != 1 { + t.Fatalf("both listed: ran %v, %d verdicts, findings %v", ran, len(run.Probes), store.findingKeys()) + } + c.Enabled = func() []string { return []string{"D13"} } + run = c.RunOnce(context.Background(), "test") + if ran["D7"] != 1 || ran["D13"] != 2 || len(run.Probes) != 1 || run.Probes[0].ID != "D13" { + t.Fatalf("D7 unlisted: ran %v, verdicts %+v", ran, run.Probes) + } + // What the unlisted probe raised stays only as long as another listed probe sees the same key (D13 raises the same + // key here); an unlisted probe alone clears its findings. + c.Enabled = func() []string { return nil } + c.RunOnce(context.Background(), "test") + if len(store.findingKeys()) != 0 { + t.Fatalf("with nothing listed, findings are %v", store.findingKeys()) + } + rows, _ := c.Probes()["probes"].([]map[string]any) + if len(rows) != 2 || rows[0]["off"] != "off by setting" || rows[1]["off"] != "off by setting" { + t.Fatalf("the verb says %v", rows) + } + c.Enabled = nil + if !c.On("anything") || !slices.Equal(DefaultProbes, []string{"D13", "D-fill"}) { + t.Fatal("no setting runs every probe, and the default list is D13 and D-fill") + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go new file mode 100644 index 00000000..a4377b73 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -0,0 +1,218 @@ +package main + +// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once, +// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the +// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"doctor", "probes"} + +// Tool names: the seat's verbs, and the module's own reads. +const ( + DoctorTool = Seat + ".doctor" + ProbesTool = Seat + ".probes" + FindingsTool = "self_check_findings" + VerdictsTool = "self_check_verdicts" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(c *Checker) []stdio.Tool { + return []stdio.Tool{ + {Name: DoctorTool, + Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " + + "failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " + + "every probe now and answers that run; probe= gives one probe's verdict whole, every finding.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "boolean", "description": "run every probe now"}, + "probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}), + Run: func(a map[string]any) (any, error) { + return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a)) + }}, + {Name: ProbesTool, + Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " + + "it raises, and for one deferred the facts it lacks on the bus.", + Run: func(map[string]any) (any, error) { return c.Probes(), nil }}, + {Name: FindingsTool, + Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " + + "finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}), + Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }}, + {Name: VerdictsTool, + Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " + + "with its counts; or one run whole by its id. The last run is the doctor verb's.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"}, + "limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}), + Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }}, + } +} + +// Doctor is the verb: the last verdict, a run now, or one probe of either. +func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) { + var r Run + switch { + case run: + r = c.RunOnce(context.Background(), why) + case c.LastRun() != nil: + r = *c.LastRun() + default: + raw, found, err := c.Store.Get(VerdictState, LastKey) + if err != nil { + return nil, fmt.Errorf("the last verdict cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+ + "every %s — run=true runs it now", FirstAfter, Every) + } + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("the last verdict is not readable: %w", err) + } + } + if probe != "" { + one, ok := oneProbe(r, probe) + if !ok { + return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool) + } + r = one + } else { + r = Overview(r) + } + return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(), + "note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " + + "longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil +} + +func oneProbe(r Run, id string) (Run, bool) { + for _, p := range r.Probes { + if strings.EqualFold(p.ID, id) { + r.Probes = []ProbeVerdict{p} + return r, true + } + } + return Run{}, false +} + +// Probes is the registry as the verb answers it. +func (c *Checker) Probes() map[string]any { + var out []map[string]any + for _, p := range c.Registry { + row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind} + if len(p.Raises) > 0 { + row["raises"] = p.Raises + } + if !c.On(p.ID) { + row["off"] = "off by setting" + } else if p.Run == nil { + row["deferred"] = p.Deferred + } + out = append(out, row) + } + return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()} +} + +// Findings are the findings written now, or one whole. +func (c *Checker) Findings(key string) (any, error) { + if key != "" { + raw, found, err := c.Store.Get(FindingsState, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no finding is written under %s now", key) + } + return json.RawMessage(raw), nil + } + written, err := c.written() + if err != nil { + return nil, fmt.Errorf("the findings cannot be read: %w", err) + } + rows := []map[string]any{} + for k, f := range written { + rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source, + "summary": f.Summary}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) }) + return map[string]any{"findings": rows, "count": len(rows)}, nil +} + +// Verdicts are the week's runs newest first, or one whole. +func (c *Checker) Verdicts(run string, limit int) (any, error) { + if run != "" { + raw, found, err := c.Store.Get(VerdictsState, run) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run) + } + return json.RawMessage(raw), nil + } + keys, err := c.Store.Keys(VerdictsState) + if err != nil { + return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err) + } + sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) }) + total := len(keys) + if limit > 0 && len(keys) > limit { + keys = keys[:limit] + } + rows := []map[string]any{} + for _, k := range keys { + raw, found, err := c.Store.Get(VerdictsState, k) + if err != nil || !found { + continue + } + var r Run + if json.Unmarshal(raw, &r) == nil { + rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts}) + } + } + return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil +} + +// runOrder orders run ids `doctor--` by time, then number. +func runOrder(id string) string { + parts := strings.Split(id, "-") + if len(parts) != 3 { + return id + } + return fmt.Sprintf("%020s-%020s", parts[1], parts[2]) +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as the seat's "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go new file mode 100644 index 00000000..a8d74293 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -0,0 +1,309 @@ +//go:build integration + +package main + +// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real +// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from +// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus, +// writes them as findings into its bucket on the bus, keeps the verdict, and says checked. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/provision.get": + // The setting `probes`: this run judges the bus probes it asserts below (D13's facts are not on this bus). + var g struct { + Kind, Key string + } + _ = json.Unmarshal(m.Params, &g) + if g.Kind == "config" && g.Key == "probes" { + p.answer(m.ID, []string{"D2", "D7", "DW", "D15"}, nil) + return + } + p.answer(m.ID, nil, fmt.Errorf("no %s value %s here", g.Kind, g.Key)) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + } + _ = json.Unmarshal(m.Params, &ask) + if ask.Key != StreamsTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + // The bus module's answer, from the server's own streams. + var detail []map[string]any + for name := range p.js.StreamNames(p.ctx).Name() { + detail = append(detail, map[string]any{"name": name}) + } + p.answer(m.ID, map[string]any{"detail": detail}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus. +func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the self-check does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{FindingsState, VerdictState, VerdictsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil { + t.Fatal(err) + } + } + checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(DoctorTool, map[string]any{"run": true}) + if err != nil { + t.Fatal(err) + } + var answer struct { + Run Run `json:"run"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + t.Fatal(err) + } + verdicts := map[string]string{} + for _, v := range answer.Run.Probes { + verdicts[v.ID] = v.Verdict + } + if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun || + verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") { + t.Errorf("the run answered %+v", answer.Run) + } + msg, err := checked.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("checked was not said: %v", err) + } + var said Run + if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run { + t.Errorf("checked said %s", msg.Data) + } + findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState) + e, err := findings.Get(ctx, "bus.EVENTS.stream-missing") + if err != nil { + t.Fatalf("the missing stream is not a finding on the bus: %v", err) + } + f, err := health.DecodeFinding(e.Value()) + if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + t.Errorf("the finding is %s", e.Value()) + } + verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) + if _, err := verdict.Get(ctx, LastKey); err != nil { + t.Errorf("the last verdict is not kept: %v", err) + } + if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) { + t.Errorf("the week's verdicts answered %s, %v", raw, err) + } +} diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod new file mode 100644 index 00000000..06acc2cf --- /dev/null +++ b/modules/mesh-self-check/go.mod @@ -0,0 +1,22 @@ +module mesh-self-check + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-self-check/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json new file mode 100644 index 00000000..e4037e98 --- /dev/null +++ b/modules/mesh-self-check/module.json @@ -0,0 +1,87 @@ +{ + "module": "mesh-self-check", + "version": "1", + "runs-as": "mesh-self-check", + "claims": [ + { + "name": "mesh-self-check", + "scope": "mesh", + "serves": [ + "doctor", + "probes" + ] + } + ], + "invokes": [ + "nats.nats_streams", + "nats.nats_closed_connections", + "seat:mesh-watchdogs.signals", + "seat:node-intrusion-prevention.banned" + ], + "reads": [ + "mesh-delivery-assigner.placement", + "mesh-delivery-assigner.seat-holders", + "mesh-delivery-watchman.presence", + "mesh-host.health", + "disk-load.readings", + "restic.good", + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.manifests" + ], + "state": [ + "findings", + "verdict", + { + "name": "verdicts", + "ttl-seconds": 604800 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-self-check", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-self-check", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-self-check" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-self-check", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-self-check", + "binary": "mesh-self-check", + "loads": [ + "mesh-self-check" + ] + } + ] + }, + "config": { + "probes": { + "type": "list", + "description": "The ids of the probes that run, each judging one thing of the mesh (mesh-self-check.probes lists them with what each asserts). A probe not listed runs nothing and raises nothing, and what it raised before is cleared. D8 and DA are in the code and off by default.", + "trusted": true, + "default": [ + "D13", + "D-fill" + ], + "runtime": true + } + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/accounts.go b/modules/mesh-vault/cmd/mesh-vault/accounts.go index e25aff28..c102337e 100644 --- a/modules/mesh-vault/cmd/mesh-vault/accounts.go +++ b/modules/mesh-vault/cmd/mesh-vault/accounts.go @@ -53,6 +53,8 @@ type AccountHash struct { Adopted bool `json:"adopted,omitempty"` From string `json:"from,omitempty"` Node string `json:"node,omitempty"` + // Rotating is a rotation under way over a second login (rotation.go, ADR 0561 §4): absent when none is. + Rotating *Rotating `json:"rotating,omitempty"` } // AccountStore is the vault's state of bus-account hashes: read and written by the vault's instances alone. @@ -61,6 +63,8 @@ type AccountStore interface { Put(a AccountHash) error // Users is every user with a key. Users() ([]string, error) + // Delete takes a user's key out: an account retired (retire.go). + Delete(user string) error } // busAccounts is AccountStore over the vault's own state through its runtime. @@ -80,6 +84,8 @@ func (busAccounts) Get(user string) (*AccountHash, error) { func (busAccounts) Users() ([]string, error) { return stdio.State(BusAccountsState).Keys() } +func (busAccounts) Delete(user string) error { return stdio.State(BusAccountsState).Delete(user) } + func (busAccounts) Put(a AccountHash) error { _, err := stdio.State(BusAccountsState).Put(a.User, a) return err @@ -181,6 +187,10 @@ func (v *Vault) HashAccount(secret string, force bool) (Hashed, error) { if err != nil { return Hashed{}, err } + if rec.Retired != nil { + v.hashes.paid(secret) + return Hashed{}, fmt.Errorf("%s is retired (%s): its hash is written again only when it is wanted again", secret, rec.Retired.Why) + } user, password, err := accountOf(secret, value) if err != nil { return Hashed{}, err @@ -259,7 +269,7 @@ func (v *Vault) HashHeld() (written []Hashed, failed map[string]string) { return nil, failed } for _, r := range records { - if !Covered(r.Name) { + if !Covered(r.Name) || r.Retired != nil { continue } h, err := v.HashAccount(r.Name, false) diff --git a/modules/mesh-vault/cmd/mesh-vault/adopt.go b/modules/mesh-vault/cmd/mesh-vault/adopt.go index 2e6e998b..5d005ca3 100644 --- a/modules/mesh-vault/cmd/mesh-vault/adopt.go +++ b/modules/mesh-vault/cmd/mesh-vault/adopt.go @@ -58,32 +58,6 @@ type adoptionKept struct { failed string // why the last look found nothing to adopt, or "" } -// busNode is the node the assignments name as holding the bus module: exactly one. -func (v *Vault) busNode() (string, error) { - if v.Assignments.Keys == nil { - return "", errors.New("the vault cannot read the assignments, so it cannot tell which node holds the bus") - } - keys, err := v.Assignments.Keys() - if err != nil { - return "", fmt.Errorf("the assignments' keys could not be read: %w", err) - } - var nodes []string - for _, k := range keys { - if node, module, ok := strings.Cut(k, "."); ok && module == BusModule { - nodes = append(nodes, node) - } - } - sort.Strings(nodes) - switch len(nodes) { - case 1: - return nodes[0], nil - case 0: - return "", fmt.Errorf("no node is assigned the bus module %s", BusModule) - } - return "", fmt.Errorf("%d nodes are assigned the bus module %s (%s): the vault adopts from one alone", len(nodes), - BusModule, strings.Join(nodes, ", ")) -} - // AdoptNow is the tool account_adopt: the bus node's have read now and adopted, though it says the same as before. func (v *Vault) AdoptNow(caller string) (Adoption, error) { if v.Haves == nil { @@ -143,6 +117,9 @@ func (v *Vault) adopt(node string, raw []byte, by string, force bool) (Adoption, if err != nil { return Adoption{}, fmt.Errorf("the have of %s on %s is not a have: %w", BusModule, node, err) } + if have.UserList != nil { + v.LoadedListHeard(have.UserList.Revision) + } if have.UserList == nil || have.UserList.Handed == nil { return Adoption{}, fmt.Errorf("the have of %s on %s hands no hashes yet (its node-engine hands them once it "+ "heard the vault's key in its answer to node-key, and writes the list)", BusModule, node) diff --git a/modules/mesh-vault/cmd/mesh-vault/askers.go b/modules/mesh-vault/cmd/mesh-vault/askers.go index e5267120..cad8a1de 100644 --- a/modules/mesh-vault/cmd/mesh-vault/askers.go +++ b/modules/mesh-vault/cmd/mesh-vault/askers.go @@ -1,6 +1,8 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/provisions" + "encoding/json" "fmt" "strings" @@ -72,8 +74,23 @@ func (l AllowList) May(caller, name string) (bool, string) { // internal/broker, KindProvisioner). const ProvisionerPrefix = "provisioner." -// AssignmentsRead is the controller's assignments mirror as this module reads it (its manifest's `reads`). -const AssignmentsRead = "mesh-controller.assignments" +// AssignmentsRead is where the vault reads each module's provisioning plan on each node (its manifest's `reads`): the +// seat mesh-provisioner's state `plans`, `.` (novox/hq ADR 0570 clause 2), no longer the controller's +// assignments. A plan is read as the assignment's value carried it (asAssignment), so the rules that read a plan are +// unchanged. +const AssignmentsRead = provisions.PlansRead + +// asAssignment is a plan's key in the form the vault's rules read: `{node, module, provisioning: plan}`, nil for none. +func asAssignment(key string, raw []byte, err error) ([]byte, error) { + if err != nil || raw == nil { + return raw, err + } + p, err := provisions.DecodePlan(raw) + if err != nil { + return nil, fmt.Errorf("the plan %s: %w", key, err) + } + return json.Marshal(map[string]any{"node": p.Node, "module": p.Module, "provisioning": p.Plan}) +} // AssignmentReader reads one value of the assignments mirror, `.`: nil when there is none. type AssignmentReader func(key string) ([]byte, error) diff --git a/modules/mesh-vault/cmd/mesh-vault/intake.go b/modules/mesh-vault/cmd/mesh-vault/intake.go index b1eb1811..b887916b 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake.go @@ -120,6 +120,15 @@ func fromTheController(caller, verb string) error { return nil } +// fromAGrantedCaller refuses a caller the runtime did not prove: `rotate` and `issued` answer every caller whose +// manifest invokes them, which the bus grants it (novox/hq ADR 0570, Consequences), no longer the controller alone. +func fromAGrantedCaller(caller, verb string) error { + if strings.TrimSpace(caller) == "" || caller == stdio.Unproven { + return fmt.Errorf("%s answers a caller the runtime proves, and it proved none", verb) + } + return nil +} + // openCall makes a key for one call and keeps its private half here until the call is taken or two minutes // have passed. func (v *Vault) openCall(verb, secret, by string, ask *AskRecord) (string, string, error) { @@ -177,6 +186,16 @@ func (v *Vault) takeCall(verb, id string) (*callKey, error) { return c, nil } +// openedBy is the caller that opened a secret-ask's call still open, as the ask's record names it; "" for none. +func (v *Vault) openedBy(id string) string { + v.callsMu.Lock() + defer v.callsMu.Unlock() + if c, ok := v.pending[id]; ok && c.verb == "secret-ask" && c.ask != nil { + return c.ask.Caller + } + return "" +} + // open opens what was sealed to the call's key, and forgets the key. func (c *callKey) open(sealed string) ([]byte, error) { defer func() { c.private = [32]byte{} }() @@ -305,15 +324,33 @@ func (v *Vault) AcceptTake(caller, call, sealed string) (Changed, error) { type AskOf struct { Node, Module, Name, Desk string // AskedBy is who asked the controller, as the bus named its caller; the controller's word, taken from the - // controller alone. + // controller alone. A caller asking directly is named by the runtime's proof, never by this (ADR 0563 §8). AskedBy string } +// askedByFor is who asked a secret-ask as the bus says it (novox/hq ADR 0563 §8): for a call relayed by the controller, +// the caller the bus named to the controller, which it takes from the controller alone, as before; for a caller the +// manifest names, that caller as the runtime proved it. Any other caller is refused: no caller names itself. +func (v *Vault) askedByFor(caller string, a AskOf) (string, error) { + switch { + case caller == ControllerCaller: + return askedByName(a.AskedBy), nil + case v.AskCallers.Allows(caller): + return askedByName(caller), nil + } + if caller == "" { + caller = stdio.Unproven + } + return "", fmt.Errorf("secret-ask answers the controller and the callers the vault's manifest names, and %s is "+ + "neither; nobody was asked to type anything", caller) +} + // AskOpen records an ask for a module's own secret before any prompt opens, holding its bounds — one open ask // per secret, three an hour (ADR 0277 §24, §25) — and answers the key the desk's prompt seals what is typed to, // and who asked as the record says it, which the prompt names (§26). func (v *Vault) AskOpen(caller string, a AskOf) (Opened, error) { - if err := fromTheController(caller, "secret-ask"); err != nil { + askedBy, err := v.askedByFor(caller, a) + if err != nil { return Opened{}, err } secret, err := OwnSecret(a.Node, a.Module, a.Name) @@ -323,7 +360,6 @@ func (v *Vault) AskOpen(caller string, a AskOf) (Opened, error) { if !namePart.MatchString(a.Desk) { return Opened{}, fmt.Errorf("nobody was asked to type anything: the desk %q is not a machine's name", a.Desk) } - askedBy := askedByName(a.AskedBy) v.Store.asksMu.Lock() defer v.Store.asksMu.Unlock() now := v.Now().UTC() @@ -378,9 +414,13 @@ var askOutcomes = map[string]bool{"dismissed": true, "timed-out": true, "empty": // AskEnd ends an ask: with what the desk sealed to its key, which the vault opens and keeps as typed, or with // the outcome the desk answered instead. How it ended is recorded whatever happens. func (v *Vault) AskEnd(caller, call, sealed, outcome string) (Changed, error) { - if err := fromTheController(caller, "secret-ask"); err != nil { + if _, err := v.askedByFor(caller, AskOf{}); err != nil { return Changed{}, err } + // An ask is ended by the caller that opened it, and by no other. + if opener := v.openedBy(call); opener != "" && opener != caller { + return Changed{}, fmt.Errorf("the ask %s was opened by %s, and %s may not end it", call, opener, caller) + } if sealed == "" && !askOutcomes[outcome] { return Changed{}, fmt.Errorf("an ask ends with what was sealed to its key, or with one of dismissed, " + "timed-out, empty, refused or failed") @@ -464,7 +504,7 @@ func askedByName(caller string) string { // 10, ADR 0305 with the vault in the controller's place): the controller asks the provider's tool, on the // provider's machine, with the call's key, and passes on what it answered. func (v *Vault) IssuedOpen(caller, provision, provider, module string) (Opened, error) { - if err := fromTheController(caller, "issued"); err != nil { + if err := fromAGrantedCaller(caller, "issued"); err != nil { return Opened{}, err } return v.openIssued(provision, provider, module) @@ -489,7 +529,7 @@ func (v *Vault) openIssued(provision, provider, module string) (Opened, error) { // IssuedTake opens what the provider's tool sealed to the call's key and keeps it as the provision's one // credential. The same value again writes nothing (ADR 0305 §5): every holder that has it is left as it is. func (v *Vault) IssuedTake(caller, call, sealed string) (Changed, error) { - if err := fromTheController(caller, "issued"); err != nil { + if err := fromAGrantedCaller(caller, "issued"); err != nil { return Changed{}, err } c, err := v.takeCall("issued", call) @@ -527,8 +567,8 @@ type Asked struct { Unchanged bool `json:"unchanged,omitempty"` } -// AskAtDesk is `secret-ask` (ADR 0349 clause 9, ADR 0277 §22–§27): asked by the controller alone, which names who -// asked it. The ask is recorded and its bounds held before any prompt opens; the vault asks the desk's prompt +// AskAtDesk is `secret-ask` (ADR 0349 clause 9, ADR 0277 §22–§27): asked by the controller, which names who asked it, +// or by a caller the vault's manifest names, named by the runtime's proof (ADR 0563 §8). The ask is recorded and its bounds held before any prompt opens; the vault asks the desk's prompt // itself, in its own name, naming the asker as recorded and giving a key made for this one call; what is typed // comes back sealed to that key and opens here. How the prompt ended is the answer and the record's. func (v *Vault) AskAtDesk(caller string, a AskOf) (Asked, error) { @@ -612,7 +652,7 @@ func (v *Vault) TakeIssued(caller, provision, provider, module, tool string) (Is // credential, as before) or one per holder (`holders`: node, module and sealed each, kept as that holder's own and // handed to it alone, perholder.go). Never both. func (v *Vault) TakeIssuedFor(caller, provision, provider, module, tool string, holders []Holder) (Issued, error) { - if err := fromTheController(caller, "issued"); err != nil { + if err := fromAGrantedCaller(caller, "issued"); err != nil { return Issued{}, err } return v.takeIssuedFor(caller, provision, provider, module, tool, holders) @@ -716,9 +756,11 @@ func (n Narrow) String() string { // value shared by every holder (ADR 0158) is one value, so it is never rotated for one consumer alone: refused, and // nothing changes. A module's own secret names no consumer. func (v *Vault) RotateFor(caller, secret string, one Narrow, by, why string) (Changed, error) { + if err := fromAGrantedCaller(caller, "rotate"); err != nil { + return Changed{}, err + } if caller != ControllerCaller { - return Changed{}, fmt.Errorf("the vault rotates for the controller alone, which knows whether the module's "+ - "secrets are filled from the vault yet; %s is not the controller: `secret rotate` at the controller", caller) + by = "" // only the controller relays who asked at its terminal; any other caller is who asked } if strings.TrimSpace(why) == "" { return Changed{}, errors.New("not rotated: a rotation says why it was asked (`why`)") diff --git a/modules/mesh-vault/cmd/mesh-vault/intake_test.go b/modules/mesh-vault/cmd/mesh-vault/intake_test.go index e795bad7..6d844cf0 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake_test.go @@ -13,6 +13,7 @@ import ( "encoding/base64" "encoding/json" "errors" + stdio "git.novox.be/novox/mesh-sdk/go" "os" "path/filepath" "strings" @@ -173,6 +174,29 @@ func TestAnAskIsBoundedToOneOpenPerSecretAndThreeAnHour(t *testing.T) { if _, err := w.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")); err != nil { t.Errorf("an ask left open past its time still held the secret: %v", err) } + // 774 (ADR 0563 §8): a caller the manifest names asks directly, is named by its proven caller and never by the + // asked_by it gives, and is held to the same bounds; one it does not name is refused. + t.Run("a named caller asks itself, within the bounds", func(t *testing.T) { + x := newVault(t, nil) + x.AskCallers = stdio.Callers{"node-tools"} + if _, err := x.AskOpen("ace.claude-code", telegramAsk("x")); err == nil || !strings.Contains(err.Error(), "ace.claude-code") { + t.Errorf("a caller the manifest does not name opened an ask: %v", err) + } + opened, err := x.AskOpen("g14.node-tools", telegramAsk("person.operator")) + if err != nil || opened.AskedBy != "g14.node-tools" { + t.Fatalf("a named caller's ask: %+v, %v", opened, err) + } + if _, err := x.AskEnd(ControllerCaller, opened.Call, "", "dismissed"); err == nil { + t.Error("another caller ended a named caller's ask") + } + if _, err := x.AskOpen("ace.node-tools", telegramAsk("")); err == nil || !strings.Contains(err.Error(), "still open (asked by g14.node-tools)") { + t.Errorf("a second ask while a named caller's is open: %v", err) + } + recorded, _ := x.Store.Asks(10) + if len(recorded) == 0 || recorded[0].AskedBy != "g14.node-tools" || recorded[0].Caller != "g14.node-tools" { + t.Errorf("recorded as %+v", recorded) + } + }) } // Only the controller reaches a desk's prompt, so only it opens and ends an ask; a value not sealed to the @@ -283,9 +307,12 @@ func TestAnIssuingProviderAnswersOnlyTheVaultsCallSealedToTheVault(t *testing.T) t.Errorf("the provider's tool answered %q", caller) } } - // The vault takes the steps from the controller alone. - if _, err := v.IssuedOpen("anchor.node-tools", "plex-api", "anchor", "plex"); err == nil { - t.Error("the vault opened an issued call for another caller") + // The vault takes the steps from a caller the runtime proved, whose manifest the bus granted the verb (ADR 0570), + // and from no unproven one. + for _, caller := range []string{"unproven", ""} { + if _, err := v.IssuedOpen(caller, "plex-api", "anchor", "plex"); err == nil { + t.Errorf("the vault opened an issued call for %q", caller) + } } changed, err := issue(t, v, provider) @@ -413,17 +440,17 @@ func TestRotateKeepsWhoAskedAndWhoCarriedIt(t *testing.T) { } } -// Rotate is the controller's alone, and says why (the review of issue 675): only the controller knows whether a -// module is filled from the vault yet, so a rotation asked by any other caller — a proven one included — would -// drift from the value the controller holds; and a rotation without a reason is refused. Neither changes anything. -func TestRotateIsTheControllersAloneAndSaysWhy(t *testing.T) { +// Rotate answers a caller the runtime proved, whose manifest the bus granted it (novox/hq ADR 0570 Consequences), and +// says why: an unproven caller, and a rotation without a reason, are refused and change nothing; a proven caller +// other than the controller is who asked, whatever `by` it says. +func TestRotateIsAProvenCallersAndSaysWhy(t *testing.T) { v := newVault(t, nil) if _, err := v.Mint("own/g14/dunst/token", "the test"); err != nil { t.Fatal(err) } - for _, caller := range []string{"g14.claude-code", "g14.node-provisioner", "unproven", ""} { + for _, caller := range []string{"unproven", ""} { if _, err := v.Rotate(caller, "own/g14/dunst/token", "jochen", "leaked"); err == nil || - !strings.Contains(err.Error(), "controller alone") { + !strings.Contains(err.Error(), "proves") { t.Errorf("%q rotated: %v", caller, err) } } @@ -436,6 +463,12 @@ func TestRotateIsTheControllersAloneAndSaysWhy(t *testing.T) { if rec, _, _ := v.Store.Read("own/g14/dunst/token"); rec.Generation != 1 { t.Fatalf("a refused rotation changed the secret: generation %d", rec.Generation) } + if _, err := v.Rotate("g14.claude-code", "own/g14/dunst/token", "jochen", "leaked in a log"); err != nil { + t.Fatal(err) + } + if rec, _, _ := v.Store.Read("own/g14/dunst/token"); rec.Generation != 2 || rec.ChangedBy != "g14.claude-code" { + t.Errorf("a session's rotation is recorded by %q at generation %d", rec.ChangedBy, rec.Generation) + } } // The names the vault keeps them under are names, never paths. diff --git a/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go b/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go index 7625ec8b..41f600e5 100644 --- a/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go @@ -48,6 +48,13 @@ func (f *fakeAccounts) Put(a AccountHash) error { return nil } +func (f *fakeAccounts) Delete(user string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.keys, user) + return nil +} + func (f *fakeAccounts) Users() ([]string, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/modules/mesh-vault/cmd/mesh-vault/lease.go b/modules/mesh-vault/cmd/mesh-vault/lease.go new file mode 100644 index 00000000..3a399d00 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/lease.go @@ -0,0 +1,49 @@ +package main + +// The switch on the controller's lease (novox/hq ADR 0561 §3): minting is on only while the lease is handed to a holder +// that is no controller, and off for a controller's value, a key deleted or expired, or one that cannot be read. The +// write that hands it away turns it on and mints every wanted account the vault holds no value for; a lease handed back +// turns it off. No clock and no setting turns it. A key's age passing is heard by no watch, so the lease is read afresh +// before every mint (LeaseRead): a lease that expired since mints nothing. + +import "git.novox.be/novox/mesh-sdk/go/buscompose" + +// LeaseHeard is the watch of the controller's lease. When minting turns on or off, every wanted account is judged again: +// minted when on, held back with the new why when off. +func (v *Vault) LeaseHeard(raw []byte, found bool, readErr error) { + if !v.setLease(raw, found, readErr) { + return + } + on, why := v.mintingOn() + v.log("minting is %s: %s", map[bool]string{true: "ON", false: "off"}[on], why) + for _, user := range v.wantedUsers() { + if w, ok := v.wanted(user); ok { + if _, err := v.mintWanted(w, "the lease write"); err != nil { + v.log("%s is not minted: %v", user, err) + } + } + } +} + +// setLease records what the lease says, and answers whether minting turned on or off by it. +func (v *Vault) setLease(raw []byte, found bool, readErr error) bool { + on, why := buscompose.LeaseHandedAway(raw, found, readErr) + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + flipped := on != m.on || m.why == "" + if flipped { + m.since = v.Now().UTC() + } + m.on, m.why = on, why + return flipped +} + +// leaseAfresh reads the lease again before a mint, where the vault can: a key that expired since is heard by no watch. +func (v *Vault) leaseAfresh() { + if v.LeaseRead == nil { + return + } + raw, found, err := v.LeaseRead() + v.setLease(raw, found, err) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 7370f0af..089c2abc 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -8,6 +8,10 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/provisions" + + "encoding/json" + "errors" "fmt" "os" @@ -50,11 +54,35 @@ func build() *Env { // Who may ask (askers.go): a node's provisioner client, for what its module's plan there names. // Where it sees a consumer bind a provider (pair.go): the same mirror, every key. // The assignments as the vault's watch keeps them (watchAssignments), read over the bus until it has caught up. - assigned = &WatchedAssignments{Fallback: readAssignment} - env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: assigned.Read, Confirm: readAssignment, Store: store}, + assigned = &WatchedAssignments{Fallback: readPlanRaw} + watchedPlan := func(key string) ([]byte, error) { raw, err := assigned.Read(key); return asAssignment(key, raw, err) } + env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: watchedPlan, Confirm: readAssignment, Store: store}, Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask, - Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, - HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }} + Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant, + HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{}, + AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), + Bound: Bound{ + Get: func(key string) ([]byte, error) { + e, err := stdio.State(BoundState).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, + Put: func(key string, value []byte) error { + _, err := stdio.State(BoundState).Put(key, json.RawMessage(value)) + return err + }, + Delete: func(key string) error { return stdio.State(BoundState).Delete(key) }, + Keys: func() ([]string, error) { return stdio.State(BoundState).Keys() }, + Provision: func(key string) ([]byte, error) { + e, err := stdio.State(provisions.ProvisionsRead).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, + }} return env } @@ -120,6 +148,74 @@ func watchHandings(env *Env) bool { return true } +// watchWants mints from the grants part's wanted accounts as they change (mint.go, ADR 0561 §2): a watch, never a poll. +// It answers whether the watch began; one that did not is begun again at the next minute pass. +func watchWants(env *Env) bool { + if env.Vault == nil { + return true + } + err := stdio.State(WantedRead).Watch("", func(c stdio.StateChange) error { + env.Vault.WantHeard(c.Key, c.Value, c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0) + return nil + }) + if err != nil { + say("the grants part's wanted accounts are not watched, so nothing is minted; begun again at the next pass: %v", err) + return false + } + // The watch returns once every current key was handed: what has no key now is no longer wanted. + env.Vault.WantsRead() + return true +} + +// placement reads the seat mesh-placement's placement through the runtime under the vault's own account (its manifest reads it). +var placement = PlacementReader{ + Keys: func() ([]string, error) { return stdio.State(PlacementRead).Keys() }, + Read: func(key string) ([]byte, error) { + e, err := stdio.State(PlacementRead).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, +} + +// readWant is one wanted account of the grants part, read afresh (issue-account). +func readWant(user string) ([]byte, bool, error) { + e, err := stdio.State(WantedRead).Get(user) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// readLease is the controller's lease key, read through the runtime under the vault's own account (its manifest reads it). +func readLease() ([]byte, bool, error) { + e, err := stdio.State(LeaseRead).Get(LeaseKey) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// watchLease turns minting on and off as the controller's lease is written (lease.go, ADR 0561 §3): a watch, never a +// poll; and the lease is read afresh before every mint. It answers whether the watch began. +func watchLease(env *Env) bool { + if env.Vault == nil { + return true + } + env.Vault.LeaseHeard(readLease()) + err := stdio.State(LeaseRead).Watch(LeaseKey, func(c stdio.StateChange) error { + deleted := c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 + env.Vault.LeaseHeard(c.Value, !deleted, nil) + return nil + }) + if err != nil { + say("the controller's lease is not watched, so minting stays as last read; begun again at the next pass: %v", err) + return false + } + return true +} + // adoptOperatorKey takes the operator key from the export the mesh writes to the vault's root (what // `operator key set` recorded; the controller keeps only a public half there), and seals again any // operator copy not yet sealed to the current key. Run every minute, so a failure partway is finished. It @@ -153,6 +249,12 @@ func adoptOperatorKey(env *Env) error { // readAssignment is one value of the controller's assignments mirror, read through the runtime under the vault's // own account (its manifest `reads` it), or nil when there is none. func readAssignment(key string) ([]byte, error) { + raw, err := readPlanRaw(key) + return asAssignment(key, raw, err) +} + +// readPlanRaw is one key of the seat mesh-provisioner's plans as it is kept, nil when there is none. +func readPlanRaw(key string) ([]byte, error) { e, err := stdio.State(AssignmentsRead).Get(key) if err != nil || e == nil { return nil, err @@ -173,7 +275,11 @@ func main() { written, failed := env.Vault.HashHeld() say("hashed %d bus accounts at start; %d could not be, tried again every minute: %v", len(written), len(failed), failed) } + // The operator key first: nothing is kept without it, so a want heard before it would not be minted. + env.pass() watched, watching := watchHandings(env), watchAssignments() + lease := watchLease(env) + wants := watchWants(env) for { if !watched { watched = watchHandings(env) @@ -181,9 +287,23 @@ func main() { if !watching { watching = watchAssignments() } + if !lease { + lease = watchLease(env) + } + if !wants { + wants = watchWants(env) + } env.pass() if env.Vault != nil { + // The signing key and authority once the operator key is taken, and `signers` when it differs (ADR 0550 §2). + if err := env.Vault.SigningPass(); err != nil { + env.Vault.signingSaid(err) + } else { + env.Vault.signingSaid(nil) + } env.Vault.HashOwed() + env.Vault.MintOwed() + env.Vault.RetirePass() } env.pairPass() if env.Custody != nil { @@ -197,3 +317,15 @@ func main() { os.Exit(1) } } + +// askCallers are the callers the vault's manifest names for secret-ask besides the controller (novox/hq ADR 0563 §8): +// MESH_VAULT_SECRET_ASK_CALLERS, comma-separated, each a module (on any machine), . or person.. +func askCallers(list string) stdio.Callers { + var out stdio.Callers + for _, c := range strings.Split(list, ",") { + if c = strings.TrimSpace(c); c != "" { + out = append(out, c) + } + } + return out +} diff --git a/modules/mesh-vault/cmd/mesh-vault/mint.go b/modules/mesh-vault/cmd/mesh-vault/mint.go new file mode 100644 index 00000000..06bb80a7 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/mint.go @@ -0,0 +1,376 @@ +package main + +// The vault mints every new bus account (novox/hq ADR 0561 §1 to §4, issue 774). +// +// The grants part says which accounts the mesh wants, in its state `accounts` that the vault alone reads (§2): one key +// per bus user, holding what the controller's credential carries beside the password. The vault watches it, and for a +// user it holds no value for and whose hash it did not adopt it makes a password, keeps the account whole (built by +// mesh-sdk's HeldCredential, byte for byte the controller's shape) as the module's own secret on that node +// (`own///|provisioner-broker>`), sealed, with the operator's copy; writes the +// bcrypt hash into `bus-accounts`; and says `account-issued`, then `secret-changed`. The node's engine asks `own` again +// on `secret-changed` and composes. +// +// **One issuer at every moment** (§3): minting is on only while the controller's lease is handed to a holder that is no +// controller (mesh-sdk LeaseHandedAway). Until then the controller mints, the vault keeps hashing what it is handed, and +// vault_status names every wanted account it holds back and why. **Nothing is minted anew** (§4): an account whose value +// the vault holds, or whose hash it adopted, is never minted; the node accounts and the controller's own are never wanted. +// **Never two values for one user** (§7): a user whose hash the vault already wrote from another of its secrets is held +// back, said, and never replaced but by a rotation. + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// WantedRead is the grants part's state of wanted accounts, as the vault's manifest reads it. +const WantedRead = buscompose.GrantsAccounts + +// LeaseRead is the controller's lease, as the vault's manifest reads it, and LeaseKey its one key. +const ( + LeaseRead = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey +) + +// AccountIssuedEvent is what the vault says when it minted an account (mesh-sdk buscompose.AccountIssued). +const AccountIssuedEvent = buscompose.AccountIssuedEvent + +// minting is the switch and what the vault knows of the wanted accounts, kept in memory and read again from the bus at +// every start: the grants part's state and the lease are the record, never this. +type minting struct { + mu sync.Mutex + on bool + why string + since time.Time + // wants are the wanted accounts, by user, as last heard. + wants map[string]buscompose.WantedAccount + // heldBack are the wanted users the vault did not mint, with why (vault_status). + heldBack map[string]string + // owed are the wanted users whose mint failed, tried again at the pass (MintOwed). + owed map[string]bool + // minted are the accounts this vault minted since it started, by user (vault_status). + minted map[string]Minted + // loadedRevision is the highest revision of the user list the bus node said it loaded (rotation.go). + loadedRevision uint64 + // retiring is what the passes keep of the accounts no longer wanted (retire.go). + retiring retiring + // mintMu keeps one mint at a time: a want heard, the switch turning on, and issue-account. + mintMu sync.Mutex +} + +// Minted is what one mint did: the account and its secret, never its value or its hash. +type Minted struct { + User string `json:"user"` + Secret string `json:"secret"` + Generation int `json:"generation"` + At time.Time `json:"at"` + // Hashed says its hash is written; false leaves it owed, written at the next pass. + Hashed bool `json:"hashed"` +} + +// MintState is what vault_status says of minting. +type MintState struct { + On bool `json:"on"` + Why string `json:"why"` + Since time.Time `json:"since,omitzero"` + Wanted int `json:"wanted"` + HeldBack map[string]string `json:"held_back"` + Minted []Minted `json:"minted"` +} + +// MintState answers how minting stands. +func (v *Vault) MintState() MintState { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + out := MintState{On: m.on, Why: m.why, Since: m.since, Wanted: len(m.wants), HeldBack: map[string]string{}, Minted: []Minted{}} + if out.Why == "" { + out.Why = "the controller's lease has not been read yet, so nothing is minted" + } + for u, why := range m.heldBack { + out.HeldBack[u] = why + } + for _, x := range m.minted { + out.Minted = append(out.Minted, x) + } + sort.Slice(out.Minted, func(i, j int) bool { return out.Minted[i].User < out.Minted[j].User }) + return out +} + +// mintingOn says whether the vault mints now, and why. +func (v *Vault) mintingOn() (bool, string) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + if v.minting.why == "" { + return false, "the controller's lease has not been read yet, so nothing is minted" + } + return v.minting.on, v.minting.why +} + +func (v *Vault) holdBack(user, why string) { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + if m.heldBack == nil { + m.heldBack = map[string]string{} + } + m.heldBack[user] = why +} + +func (v *Vault) noLongerHeld(user string) { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + delete(m.heldBack, user) +} + +// AccountSecret is the vault's name of the own secret a wanted account is held in. +func AccountSecret(w buscompose.WantedAccount) (string, error) { + return OwnSecret(w.Node, w.Module, w.OwnSecret()) +} + +// WantHeard is the watch of the grants part's wanted accounts: a key put is a want, minted when minting is on; a key +// deleted is a want gone (retire.go). +func (v *Vault) WantHeard(user string, raw []byte, deleted bool) { + if deleted { + v.wantGone(user) + return + } + w, err := buscompose.ReadWantedAccount(user, raw) + if err != nil { + v.log("the wanted account %s is not read, so nothing is minted for it: %v", user, err) + v.holdBack(user, "its wanted key is not readable: "+err.Error()) + return + } + v.minting.mu.Lock() + if v.minting.wants == nil { + v.minting.wants = map[string]buscompose.WantedAccount{} + } + v.minting.wants[user] = w + v.minting.mu.Unlock() + v.wantBack(user) + if _, err := v.mintWanted(w, "the grants part's want of "+user); err != nil { + v.log("%s is not minted: %v", user, err) + } +} + +// errHeldBack is a want the vault does not mint now, with why: not an error of the vault. +type errHeldBack struct{ why string } + +func (e errHeldBack) Error() string { return e.why } + +// mintWanted mints w when minting is on, the vault holds no value for it and adopted no hash of it; otherwise it says +// why not and, where that is a hold, keeps it for vault_status. +func (v *Vault) mintWanted(w buscompose.WantedAccount, by string) (Minted, error) { + m, err := v.mintOnce(w, by) + var held errHeldBack + v.owe(w.User, err != nil && !errors.As(err, &held) && w.Valid() == nil) + return m, err +} + +// mintOnce is one try of mintWanted. +func (v *Vault) mintOnce(w buscompose.WantedAccount, by string) (Minted, error) { + v.minting.mintMu.Lock() + defer v.minting.mintMu.Unlock() + if err := w.Valid(); err != nil { + v.holdBack(w.User, err.Error()) + return Minted{}, err + } + secret, err := AccountSecret(w) + if err != nil { + v.holdBack(w.User, err.Error()) + return Minted{}, err + } + if _, held, err := v.Store.Read(secret); err != nil { + return Minted{}, err + } else if held { + // Its value is held (the controller's, handed in, or one the vault minted): never minted anew (§4). + v.noLongerHeld(w.User) + return Minted{}, errHeldBack{secret + " is held already: an account the vault holds a value for is never minted anew (ADR 0561 §4)"} + } + if v.Accounts == nil { + v.holdBack(w.User, "the vault has no state of bus accounts to write the hash to") + return Minted{}, errors.New("the vault has no state of bus accounts to write the hash to") + } + was, err := v.Accounts.Get(w.User) + if err != nil { + v.holdBack(w.User, "its key in "+BusAccountsState+" cannot be read: "+err.Error()) + return Minted{}, err + } + if was != nil && was.Adopted { + // The hash the bus loaded is kept (§4): not minted, and not held back either. + v.noLongerHeld(w.User) + return Minted{}, errHeldBack{w.User + "'s hash was adopted from the list the bus loaded: it keeps it, and is never minted (ADR 0561 §4)"} + } + if was != nil && was.Of != "" && was.Of != secret { + why := fmt.Sprintf("%s holds the hash of %s for %s, not of %s: a user has one value, so nothing is minted until "+ + "a person sees why (ADR 0561 §7)", BusAccountsState, was.Of, w.User, secret) + v.holdBack(w.User, why) + return Minted{}, errHeldBack{why} + } + v.leaseAfresh() + if on, why := v.mintingOn(); !on { + v.holdBack(w.User, "held back: "+why) + return Minted{}, errHeldBack{why} + } + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return Minted{}, err + } + value, err := buscompose.HeldCredential(w, base64.RawURLEncoding.EncodeToString(raw)) + if err != nil { + v.holdBack(w.User, err.Error()) + return Minted{}, err + } + c, err := v.keepLocked(secret, string(value), "the vault", "", OriginMade, keepOptions{neverReplace: true, quiet: true}) + if err != nil { + v.holdBack(w.User, "its value could not be kept: "+err.Error()) + return Minted{}, err + } + out := Minted{User: w.User, Secret: secret, Generation: c.Generation, At: c.At} + // The hash first, then account-issued, then secret-changed: the grants part composes the user before the node asks. + if _, err := v.HashAccount(secret, false); err != nil { + v.hashes.owe(secret, err.Error()) + v.log("%s is minted and kept, and ITS HASH IS NOT WRITTEN, written at the next pass: %v", w.User, err) + } else { + out.Hashed = true + if v.Emit != nil { + e := buscompose.AccountIssued{User: w.User, Kind: w.Kind, Node: w.Node, Module: w.Module, Process: w.Process, + Generation: c.Generation, At: c.At} + body := map[string]any{} + b, _ := json.Marshal(e) + _ = json.Unmarshal(b, &body) + if err := v.Emit(AccountIssuedEvent, body); err != nil { + v.log("%s is minted and hashed, but %s could not be said: %v", w.User, AccountIssuedEvent, err) + } + } + } + v.announce(c) + v.minting.mu.Lock() + if v.minting.minted == nil { + v.minting.minted = map[string]Minted{} + } + v.minting.minted[w.User] = out + v.minting.mu.Unlock() + v.noLongerHeld(w.User) + v.log("minted the bus account %s as %s (generation %d), for %s", w.User, secret, c.Generation, by) + return out, nil +} + +// MintOwed tries again, at the vault's minute pass, every wanted account whose mint failed (its value could not be kept, +// its key read, its password made): a want heard is never lost to one refusal. A want held back is not owed: the switch +// or the want changing judges it again. +func (v *Vault) MintOwed() { + v.minting.mu.Lock() + var owed []string + for u := range v.minting.owed { + owed = append(owed, u) + } + v.minting.mu.Unlock() + sort.Strings(owed) + for _, u := range owed { + if w, ok := v.wanted(u); ok { + if _, err := v.mintWanted(w, "the pass, after a mint that failed"); err != nil { + v.log("%s is still not minted: %v", u, err) + } + } else { + v.owe(u, false) + } + } +} + +// owe marks user's mint as failed (tried again at the pass) or not. +func (v *Vault) owe(user string, owed bool) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + if !owed { + delete(v.minting.owed, user) + return + } + if v.minting.owed == nil { + v.minting.owed = map[string]bool{} + } + v.minting.owed[user] = true +} + +// heldBackOrMinted is a mint's answer as a tool gives it. +func heldBackOrMinted(m Minted, err error) (any, error) { + var held errHeldBack + if errors.As(err, &held) { + return map[string]any{"minted": false, "why": held.why}, nil + } + if err != nil { + return nil, err + } + return map[string]any{"minted": true, "account": m}, nil +} + +// IssueAccount is the seat's verb issue-account (ADR 0561 §9): one wanted user minted now, its key read afresh from the +// grants part's state. The recovery when a want was not heard: it mints nothing a want would not, and is refused while +// minting is off, for a node's account and the controller's (never minted here, §4), and for a user nobody wants. +func (v *Vault) IssueAccount(caller, user string) (any, error) { + user = strings.TrimSpace(user) + if user == "" { + return nil, errors.New("issue-account names a bus user (`user`), e.g. ace.mesh-build-go") + } + if nodeOrController(user) { + return nil, fmt.Errorf("%s is never minted by the vault: a node's account and the controller's keep the hash they "+ + "have (ADR 0561 §4)", user) + } + v.leaseAfresh() + if on, why := v.mintingOn(); !on { + return nil, fmt.Errorf("not minted: %s", why) + } + raw, found, err := []byte(nil), false, error(nil) + if v.WantRead != nil { + raw, found, err = v.WantRead(user) + } else if w, ok := v.wanted(user); ok { + raw, err = json.Marshal(w) + found = err == nil + } + if err != nil { + return nil, fmt.Errorf("the wanted account %s could not be read: %w", user, err) + } + if !found { + return nil, fmt.Errorf("the grants part holds no wanted account %s: the vault mints only what placement composes "+ + "(grants_accounts lists them)", user) + } + w, err := buscompose.ReadWantedAccount(user, raw) + if err != nil { + return nil, err + } + v.log("issue-account %s, asked by %s", user, someoneOr("", caller)) + return heldBackOrMinted(v.mintWanted(w, "issue-account by "+someoneOr("", caller))) +} + +// wantedUsers is every wanted user heard, sorted. +func (v *Vault) wantedUsers() []string { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + out := make([]string, 0, len(v.minting.wants)) + for u := range v.minting.wants { + out = append(out, u) + } + sort.Strings(out) + return out +} + +func (v *Vault) wanted(user string) (buscompose.WantedAccount, bool) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + w, ok := v.minting.wants[user] + return w, ok +} + +// nodeOrController says user is a node's account or the controller's: never minted here (ADR 0561 §4, §9). +func nodeOrController(user string) bool { + return user == ControllerCaller || strings.HasPrefix(user, "node.") +} diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go new file mode 100644 index 00000000..133c3131 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -0,0 +1,422 @@ +//go:build integration + +package main + +// The vault whole against a real bus (novox/hq ADR 0561 §1 to §4, issue 774's acceptance 12): the built binary, played +// over its real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* +// and mesh/publish answered from a real nats-server's key-value buckets and subjects), its store in a directory of its +// own. A want written to the grants part's state is minted only once the controller's lease is handed away: the hash is +// in bus-accounts, own answers the value sealed to the node, the value is the controller's shape and its hash's +// password, account-issued is said; an adopted user is never minted, and a lease handed back mints nothing. + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/nacl/box" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/seal" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// wire is one line between the runtime and the bundle (the SDK's stdio message). +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime as far as the vault needs it, over a real bus. +type player struct { + t *testing.T + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + said []string +} + +// bucketOf is the bucket a state names: the vault's own by its local name, another's as `.`. +func bucketOf(state string) string { + if owner, name, ok := strings.Cut(state, "."); ok { + return owner + "_" + name + } + return "mesh-vault_" + state +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + return p.js.KeyValue(p.ctx, bucketOf(state)) +} + +// serve answers what the bundle asks, and hands the answers to what the test asked it. +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + rev, err = kv.Put(p.ctx, a.Key, a.Value) + if err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := strings.ReplaceAll(a.Key, "**", ">") + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.mu.Lock() + p.said = append(p.said, e.Key+" "+string(e.Body)) + p.mu.Unlock() + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.mesh-vault.event."+e.Key, e.Body)) + default: + // mesh/ask and the rest: nothing answers on this bus, as a call with no responder. + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +// call is a tools/call of the vault as caller, answered as the tool's JSON. +func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(30 * time.Second): + return nil, fmt.Errorf("%s did not answer in 30s", name) + } +} + +func (p *player) put(t *testing.T, state, key string, v any) { + t.Helper() + kv, err := p.kv(state) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(v) + if _, err := kv.Put(p.ctx, key, raw); err != nil { + t.Fatal(err) + } +} + +func (p *player) hashOf(t *testing.T, user string) *AccountHash { + t.Helper() + kv, err := p.kv(BusAccountsState) + if err != nil { + t.Fatal(err) + } + e, err := kv.Get(p.ctx, user) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + if err != nil { + t.Fatal(err) + } + var a AccountHash + if err := json.Unmarshal(e.Value(), &a); err != nil { + t.Fatal(err) + } + return &a +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +// 774/AC12. +func TestTheVaultWholeMintsOnTheBusOnlyFromTheLeaseWrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, "mesh-vault") + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the vault does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{"mesh-vault_bus-accounts", "mesh-grants_accounts", "mesh-controller_lease", + "mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil { + t.Fatal(err) + } + } + // The operator key, as the mesh exports it to the vault's root. + opPub, _, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(dir, "root") + _ = os.MkdirAll(root, 0o700) + export, _ := json.Marshal(map[string]string{"operator-key": base64.StdEncoding.EncodeToString(opPub[:])}) + if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"), + "MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"), + "MESH_NODE=novox", "MESH_MODULE=mesh-vault") + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + status := func() MintState { + raw, err := p.call("vault_status", "person.jochen", nil) + if err != nil { + return MintState{} + } + var s struct { + Minting MintState `json:"minting"` + } + _ = json.Unmarshal(raw, &s) + return s.Minting + } + // The controller holds the lease: a want is held back, nothing is minted. + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"instance": "novox/1/2026-10-27", "epoch": 57}) + x, y, z := aWant("mod-x"), aWant("mod-y"), aWant("mod-z") + p.put(t, "mesh-vault.bus-accounts", y.User, AccountHash{User: y.User, Hash: "$2a$04$adoptedadoptedadoptedadoptedadoptedadoptedadopte", Adopted: true}) + p.put(t, "mesh-grants.accounts", x.User, x) + p.put(t, "mesh-grants.accounts", y.User, y) + eventually(t, "the want held back under the controller's lease", func() bool { + return strings.Contains(status().HeldBack[x.User], "controller holds the lease") + }) + if p.hashOf(t, x.User) != nil { + t.Fatal("minted under the controller's lease") + } + // The write that hands the lease away mints it. + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + eventually(t, "the account minted at the lease write", func() bool { return p.hashOf(t, x.User) != nil }) + nodePub, nodePriv := aSealingKey(t) + if _, err := p.call(Seat+".node-key", "node.ace", map[string]any{"key": nodePub}); err != nil { + t.Fatal(err) + } + raw, err := p.call(Seat+".own", "node.ace", map[string]any{}) + if err != nil { + t.Fatal(err) + } + var own OwnAnswer + if err := json.Unmarshal(raw, &own); err != nil { + t.Fatal(err) + } + if len(own.Own) != 1 || own.Own[0].Module != "mod-x" || own.Own[0].Name != "broker" || own.Own[0].Origin != OriginMade { + t.Fatalf("own answers %+v", own.Own) + } + value, err := seal.Open(own.Own[0].Sealed, nodePriv) + if err != nil { + t.Fatal(err) + } + var held struct{ Password string } + _ = json.Unmarshal([]byte(value), &held) + shape, _ := buscompose.HeldCredential(x, held.Password) + if value != string(shape) { + t.Errorf("the minted value %q is not the controller's shape %q", value, shape) + } + if h := p.hashOf(t, x.User); bcrypt.CompareHashAndPassword([]byte(h.Hash), []byte(held.Password)) != nil || h.Adopted || h.Generation != 1 { + t.Errorf("bus-accounts %s holds %+v, not the minted password's hash", x.User, h) + } + if h := p.hashOf(t, y.User); h == nil || !h.Adopted { + t.Errorf("the adopted %s was replaced: %+v", y.User, h) + } + p.mu.Lock() + said := strings.Join(p.said, "\n") + p.mu.Unlock() + if !strings.Contains(said, AccountIssuedEvent+` {"`) || !strings.Contains(said, `"user":"ace.mod-x"`) || strings.Contains(said, held.Password) { + t.Errorf("the vault said: %s", said) + } + // Handed back to a controller: a new want mints nothing. + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"instance": "novox/2/2026-10-28", "epoch": 58}) + p.put(t, "mesh-grants.accounts", z.User, z) + eventually(t, "the new want held back once the lease is the controller's again", func() bool { + return strings.Contains(status().HeldBack[z.User], "controller holds the lease") + }) + if p.hashOf(t, z.User) != nil { + t.Error("minted after the lease was handed back") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_test.go new file mode 100644 index 00000000..e36454e5 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/mint_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// aWant is a wanted account of a module on ace, as the grants part writes it. +func aWant(module string) buscompose.WantedAccount { + return buscompose.WantedAccount{User: "ace." + module, Kind: buscompose.KindModule, Node: "ace", Module: module, + URL: "nats://novox.mesh:4222", Fingerprint: "ab:cd", Claims: []buscompose.ClaimedSeat{{Seat: "s", Scope: "node"}}} +} + +func wantRaw(t *testing.T, w buscompose.WantedAccount) []byte { + t.Helper() + raw, err := json.Marshal(w) + if err != nil { + t.Fatal(err) + } + return raw +} + +// mintingVault is a vault with a state of bus accounts, ace's sealing key stated, ace's modules assigned, and minting +// turned on or off as on says. +func mintingVault(t *testing.T, on bool) (testVault, *fakeAccounts, string) { + t.Helper() + plans := assignments{"ace.mod-x": `{"node":"ace","module":"mod-x"}`, "ace.mod-y": `{"node":"ace","module":"mod-y"}`} + v := newVault(t, nil) + v.Askers = Declared{Read: plans.read, Store: v.Store} + accounts := &fakeAccounts{} + v.Accounts = accounts + pub, priv := aSealingKey(t) + if err := v.recordNodeKey("ace", pub, "node.ace"); err != nil { + t.Fatal(err) + } + if on { + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + } else { + v.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil) + } + return v, accounts, priv +} + +// 774/AC7 (novox/hq ADR 0561 §1, §2, §4): a want with no value, minting on, is minted once: kept whole in the controller's +// shape as the module's broker, its hash in bus-accounts, said account-issued before secret-changed, and answered by own +// sealed to the node; a value held or a hash adopted is never minted. +func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) { + v, accounts, nodePriv := mintingVault(t, true) + w := aWant("mod-x") + before := len(v.bus.events) + v.WantHeard(w.User, wantRaw(t, w), false) + rec, held, err := v.Store.Read("own/ace/mod-x/broker") + if err != nil || !held || rec.Generation != 1 || rec.Origin != OriginMade { + t.Fatalf("the minted account is kept as %+v (held %v, %v)", rec, held, err) + } + value, err := seal.Open(rec.AtRest, v.Own.PrivateKey) + if err != nil { + t.Fatal(err) + } + var got struct{ Password string } + _ = json.Unmarshal([]byte(value), &got) + shape, _ := buscompose.HeldCredential(w, got.Password) + if len(got.Password) != 40 || value != string(shape) { + t.Errorf("the minted value is %q, not the controller's shape %q", value, shape) + } + if !accounts.hashOf(w.User, got.Password, 1) { + t.Errorf("%s's hash is not the minted password's: %+v", w.User, accounts.keys[w.User]) + } + var order []string + for _, e := range v.bus.events[before:] { + order = append(order, e["event"].(string)) + } + if strings.Join(order, ",") != AccountHashedEvent+","+AccountIssuedEvent+","+ChangedEvent { + t.Errorf("said %v, want the hash, account-issued, then secret-changed", order) + } + for _, raw := range v.bus.raw[before:] { + if strings.Contains(raw, got.Password) || strings.Contains(raw, "$2a$") { + t.Errorf("a value or a hash was said: %s", raw) + } + } + own, err := v.OwnOfNode("node.ace") + if err != nil || len(own.Own) != 1 { + t.Fatalf("own answered %+v, %v", own, err) + } + if opened, err := seal.Open(own.Own[0].Sealed, nodePriv); err != nil || opened != value { + t.Errorf("own's sealed value opens to %q, %v", opened, err) + } + + // Heard again, or for a value held, or for a hash adopted: nothing is minted, nothing is held back. + v.WantHeard(w.User, wantRaw(t, w), false) + if rec, _, _ := v.Store.Read("own/ace/mod-x/broker"); rec.Generation != 1 { + t.Errorf("a want heard again minted generation %d", rec.Generation) + } + accounts.keys["ace.mod-y"] = AccountHash{User: "ace.mod-y", Hash: "$2a$04$adopted", Adopted: true} + y := aWant("mod-y") + v.WantHeard(y.User, wantRaw(t, y), false) + if _, held, _ := v.Store.Read("own/ace/mod-y/broker"); held || accounts.keys["ace.mod-y"].Hash != "$2a$04$adopted" { + t.Errorf("an adopted user was minted") + } + if st := v.MintState(); len(st.HeldBack) != 0 || len(st.Minted) != 1 { + t.Errorf("vault_status says %+v: an adopted user is not held back, one account minted", st) + } +} + +// 774/AC8 (novox/hq ADR 0561 §3): while the controller holds the lease nothing is minted and vault_status says why and +// names each wanted account held back (never an adopted one); the write that hands the lease away mints them; a lease +// handed back, expired or unreadable turns minting off again, read afresh before every mint. +func TestNothingIsMintedUntilTheLeaseIsHandedAway(t *testing.T) { + v, accounts, _ := mintingVault(t, false) + accounts.keys = map[string]AccountHash{"ace.mod-y": {User: "ace.mod-y", Hash: "$2a$04$adopted", Adopted: true}} + for _, m := range []string{"mod-x", "mod-y"} { + w := aWant(m) + v.WantHeard(w.User, wantRaw(t, w), false) + } + st := v.MintState() + if st.On || !strings.Contains(st.Why, "controller holds the lease") || len(st.HeldBack) != 1 || + !strings.Contains(st.HeldBack["ace.mod-x"], "controller holds the lease") { + t.Fatalf("under the controller's lease vault_status says %+v", st) + } + if _, held, _ := v.Store.Read("own/ace/mod-x/broker"); held { + t.Fatal("minted while the controller holds the lease") + } + // Handed away: the wanted account is minted at that write, the adopted one is not. + lease := []byte(`{"handed-to":"the operator's hand-over"}`) + v.LeaseRead = func() ([]byte, bool, error) { return lease, true, nil } + v.LeaseHeard(lease, true, nil) + if _, held, _ := v.Store.Read("own/ace/mod-x/broker"); !held { + t.Fatalf("not minted at the lease write: %+v", v.MintState()) + } + if _, held, _ := v.Store.Read("own/ace/mod-y/broker"); held { + t.Error("an adopted account was minted at the lease write") + } + if st := v.MintState(); !st.On || len(st.HeldBack) != 0 { + t.Errorf("after the lease write vault_status says %+v", st) + } + // The key expired (no watch hears a key's age pass): read afresh before the next mint, nothing is minted. + v.LeaseRead = func() ([]byte, bool, error) { return nil, false, nil } + z := aWant("mod-z") + v.WantHeard(z.User, wantRaw(t, z), false) + if _, held, _ := v.Store.Read("own/ace/mod-z/broker"); held { + t.Error("minted after the lease's key expired") + } + if st := v.MintState(); st.On || !strings.Contains(st.HeldBack["ace.mod-z"], "no lease is written") { + t.Errorf("after the key expired vault_status says %+v", st) + } +} + +// 774/AC10 issue-account (novox/hq ADR 0561 §9): the verb mints one wanted user now, read afresh; it is refused while +// minting is off, for a node's account and the controller's, and for a user nobody wants; a user held already is said so. +func TestIssueAccountMintsOneWantedUserNowAndRefusesTheRest(t *testing.T) { + v, _, _ := mintingVault(t, false) + w := aWant("mod-x") + raw := wantRaw(t, w) + v.WantRead = func(user string) ([]byte, bool, error) { + if user == w.User { + return raw, true, nil + } + return nil, false, nil + } + for user, says := range map[string]string{ + "ace.mod-x": "controller holds the lease", + "node.ace": "never minted", + "controller": "never minted", + } { + if _, err := v.IssueAccount("person.jochen", user); err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("%s: %v, want a refusal saying %q", user, err, says) + } + } + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + if _, err := v.IssueAccount("person.jochen", "ace.nobody"); err == nil || !strings.Contains(err.Error(), "no wanted account") { + t.Errorf("a user nobody wants: %v", err) + } + got, err := v.IssueAccount("person.jochen", w.User) + if err != nil || got.(map[string]any)["minted"] != true { + t.Fatalf("issue-account: %v, %v", got, err) + } + if again, err := v.IssueAccount("person.jochen", w.User); err != nil || again.(map[string]any)["minted"] != false || + !strings.Contains(again.(map[string]any)["why"].(string), "held already") { + t.Errorf("issue-account again: %v, %v", again, err) + } +} + +// 774/AC10 the rotation's rule (novox/hq ADR 0561 §4): at most one rotation begins per revision of the loaded user list, +// over the second login `--b`, never of a node's account, the controller's or an adopted hash, and `rotate` of a +// covered account stays refused, so nothing rotates in place. +func TestOneRotationBeginsPerRevisionOfTheLoadedList(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + for _, m := range []string{"mod-x", "mod-y"} { + w := aWant(m) + v.WantHeard(w.User, wantRaw(t, w), false) + } + if _, err := v.BeginRotation("ace.mod-x"); err == nil || !strings.Contains(err.Error(), "not known yet") { + t.Errorf("began with no loaded revision known: %v", err) + } + v.LoadedListHeard(570) + r, err := v.BeginRotation("ace.mod-x") + if err != nil || r.Login != "ace.mod-x--b" || r.Revision != 570 || r.Generation != 2 || accounts.keys["ace.mod-x"].Rotating == nil { + t.Fatalf("the first rotation: %+v, %v", r, err) + } + if _, err := v.BeginRotation("ace.mod-y"); err == nil || !strings.Contains(err.Error(), "one rotation begins per revision") { + t.Errorf("a second rotation in one revision: %v", err) + } + v.LoadedListHeard(571) + if _, err := v.BeginRotation("ace.mod-y"); err != nil { + t.Errorf("the next revision's rotation: %v", err) + } + if _, err := v.BeginRotation("ace.mod-x"); err == nil || !strings.Contains(err.Error(), "rotating already") { + t.Errorf("an account rotating began again: %v", err) + } + accounts.keys["node.ace"] = AccountHash{User: "node.ace", Hash: "$2a$04$n", Adopted: true} + for _, u := range []string{"node.ace", "controller"} { + if _, err := v.BeginRotation(u); err == nil { + t.Errorf("%s began a rotation", u) + } + } + if _, err := v.RotateFor(ControllerCaller, "own/ace/mod-x/broker", Narrow{}, "the test", "a rotation in place"); err == nil { + t.Error("a covered account rotated in place") + } +} + +// A mint that failed (here: no operator key yet, so nothing can be kept) is tried again at the pass, never lost. +func TestAFailedMintIsTriedAgainAtThePass(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + key, _ := v.Store.OperatorKey() + if err := v.Store.setOperatorKey("", "the test", v.Now()); err != nil { + t.Fatal(err) + } + w := aWant("mod-x") + v.WantHeard(w.User, wantRaw(t, w), false) + if _, ok := accounts.keys[w.User]; ok { + t.Fatal("minted with no operator key") + } + if err := v.Store.setOperatorKey(key, "the test", v.Now()); err != nil { + t.Fatal(err) + } + v.MintOwed() + if _, ok := accounts.keys[w.User]; !ok { + t.Errorf("not minted at the pass after the failure: %+v", v.MintState()) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/outcomes.go b/modules/mesh-vault/cmd/mesh-vault/outcomes.go new file mode 100644 index 00000000..8f4db79a --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/outcomes.go @@ -0,0 +1,174 @@ +package main + +// The vault's outcome for each credential it provides (novox/hq ADR 0570 clauses 3, 6, 7): an entry in its own state +// `bound`, one per credential provision `..secret-[.]`, which the holder of +// mesh-provisioner reads to set that key's state. The entry names the value by the vault's own name for it +// (provisions.VaultName, as its verb `own` names it) and its generation, never the value. A value the vault holds is +// provided; one only the provider's software issues, or only a person gives, waits for them; one the vault has not +// made yet has no entry, and the key stays asked. An offer whose form is none has no credential and no entry. An entry +// is let go only once its key is retiring with the operator's warrant, or gone. + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// BoundState is the vault's state of outcome entries. +const BoundState = "bound" + +// Bound is the vault's own state `bound`, and what it reads of the seat's provisions. +type Bound struct { + Get func(key string) ([]byte, error) + Put func(key string, value []byte) error + Delete func(key string) error + Keys func() ([]string, error) + // Provision is one key of mesh-provisioner's provisions, nil for none. + Provision func(key string) ([]byte, error) +} + +// BoundOutcome is one entry of `bound`, in the provider helper's entry layout with the generation beside it. +type BoundOutcome struct { + Provider string `json:"provider"` + ProviderModule string `json:"provider-module"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + VaultName string `json:"vault-name,omitempty"` + State string `json:"state"` + Why string `json:"why,omitempty"` + Generation int `json:"generation,omitempty"` +} + +// OutcomePass writes each credential's outcome that differs and lets go each entry no plan names whose key is retiring +// with the operator's warrant or gone. It answers what it wrote and removed. +func (v *Vault) OutcomePass() ([]string, error) { + if v.Bound.Get == nil || v.Assignments.Keys == nil { + return nil, nil + } + want, err := v.outcomes() + if err != nil { + return nil, err + } + var said []string + for _, key := range sortedOutcomeKeys(want) { + raw, _ := json.Marshal(want[key]) + if have, err := v.Bound.Get(key); err == nil && bytes.Equal(have, raw) { + continue + } + if err := v.Bound.Put(key, raw); err != nil { + return said, fmt.Errorf("the outcome %s: %w", key, err) + } + said = append(said, key+" "+want[key].State) + } + keys, err := v.Bound.Keys() + if err != nil { + return said, err + } + for _, key := range keys { + if _, still := want[key]; still { + continue + } + if !v.letGo(key) { + continue + } + if err := v.Bound.Delete(key); err != nil { + return said, err + } + said = append(said, key+" let go") + } + return said, nil +} + +// letGo says an entry no plan names may go: its key is gone, or retiring with the operator's Approve. +func (v *Vault) letGo(key string) bool { + if v.Bound.Provision == nil { + return false + } + raw, err := v.Bound.Provision(key) + if err != nil { + return false + } + if raw == nil { + return true + } + p, err := provisions.Decode(raw) + return err == nil && p.State == provisions.Retiring && p.Warrant != nil && p.Warrant.Option == "approve" +} + +// outcomes is every credential's outcome the plans call for, by its key. +func (v *Vault) outcomes() (map[string]BoundOutcome, error) { + keys, err := v.Assignments.Keys() + if err != nil { + return nil, fmt.Errorf("the plans' keys: %w", err) + } + out := map[string]BoundOutcome{} + for _, k := range keys { + raw, err := v.Assignments.Read(k) + if err != nil { + return nil, fmt.Errorf("the plan %s: %w", k, err) + } + if raw == nil { + continue + } + var a plannedProvisions + if err := json.Unmarshal(raw, &a); err != nil || a.Provisioning == nil { + continue + } + consumer := provisions.End{Module: a.Module, Node: a.Node} + for _, p := range a.Provisioning.Provisions { + provider := provisions.End{Module: p.ProviderModule, Node: p.Provider} + if p.ProviderModule == provisions.VaultModule && p.Provision == "secret" { + // The vault's own offer: a value it hands the consumer's client on request (`get`), which it serves as + // soon as it runs. + key, err := provisions.Key(a.Node, a.Module, p.Provision, p.Local) + if err == nil { + out[key] = BoundOutcome{Provider: p.Provider, ProviderModule: provisions.VaultModule, Provision: p.Provision, + Consumer: a.Node, Module: a.Module, Local: p.Local, State: provisions.Provided, + Why: "the vault hands it to the consumer's client on request"} + } + continue + } + ck, ok := provisions.CredentialKey(p.CredentialForm, consumer, provider, p.Provision, p.Local) + if !ok { + continue // form none: no credential, nothing minted + } + parent := provisions.Provision{Provision: p.Provision, Local: p.Local, Consumer: consumer, Provider: provider} + name := provisions.VaultName(p.CredentialForm, parent) + cn, cm, cp, cl, err := provisions.ParseKey(ck) + if err != nil || name == "" { + continue + } + o := BoundOutcome{Provider: p.Provider, ProviderModule: provisions.VaultModule, Provision: cp, Consumer: cn, + Module: cm, Local: cl, VaultName: name} + rec, held, err := v.Store.Read(name) + switch { + case err != nil: + o.State, o.Why = provisions.Failed, "the vault cannot read "+name+": "+err.Error() + case held: + o.State, o.Generation = provisions.Provided, rec.Generation + case p.CredentialForm == FormIssued: + o.State, o.Why = provisions.Waits, "the provider's software issues it (ADR 0305): `issued` brings it in" + case p.CredentialForm == FormGiven: + o.State, o.Why = provisions.Waits, "a person gives it (ADR 0337): `accept` brings it in" + default: + continue // not made yet: the key stays asked until the pair pass makes it + } + out[ck] = o + } + } + return out, nil +} + +func sortedOutcomeKeys(m map[string]BoundOutcome) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-vault/cmd/mesh-vault/outcomes_test.go b/modules/mesh-vault/cmd/mesh-vault/outcomes_test.go new file mode 100644 index 00000000..29bae5f9 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/outcomes_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "encoding/json" + "slices" + "sort" + "strings" + "testing" +) + +// The vault says each credential's outcome in its own state `bound`, by its own name for the value and its generation, +// never the value: held is provided, an issued or given value not yet held waits, one not made yet has no entry, an +// offer of form none has none, the vault's own offer is provided; an entry no plan names goes only once its key is gone +// or retiring with the operator's Approve (novox/hq ADR 0570 clauses 3, 5-7; issue 1954 V1). +func TestTheVaultSaysEachCredentialsOutcome(t *testing.T) { + v := newVault(t, NoAsker{}) + plan := func(node, module string, provisions ...map[string]any) []byte { + raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{"provisions": provisions}}) + return raw + } + entry := func(provision, provider, module, form string) map[string]any { + return map[string]any{"provision": provision, "provider": provider, "provider-module": module, "credential-form": form} + } + plans := map[string][]byte{ + "g14.gitea": plan("g14", "gitea", + entry("postgres-database", "ace", "postgres", "pair"), + entry("oidc-client", "novox", "keycloak", "shared"), + entry("media-token", "ace", "jellyfin", "issued"), + entry("bot-token", "ace", "telegram", "given"), + entry("route", "novox", "route-adapter", "none"), + entry("secret", "novox", "mesh-vault", "pair")), + } + v.Assignments = Mirror{ + Keys: func() ([]string, error) { + var k []string + for key := range plans { + k = append(k, key) + } + return k, nil + }, + Read: func(key string) ([]byte, error) { return plans[key], nil }} + bound := map[string][]byte{} + keys := map[string]string{} // the seat's provisions, by key: a state, or a raw value + v.Bound = Bound{ + Get: func(key string) ([]byte, error) { return bound[key], nil }, + Put: func(key string, value []byte) error { bound[key] = value; return nil }, + Delete: func(key string) error { delete(bound, key); return nil }, + Keys: func() ([]string, error) { + var out []string + for k := range bound { + out = append(out, k) + } + sort.Strings(out) + return out, nil + }, + Provision: func(key string) ([]byte, error) { + if raw, ok := keys[key]; ok { + return []byte(raw), nil + } + return nil, nil + }, + } + if _, err := v.Put("pair/ace/postgres-database/g14/gitea", "pw", "the test"); err != nil { + t.Fatal(err) + } + if _, err := v.OutcomePass(); err != nil { + t.Fatal(err) + } + read := func(key string) BoundOutcome { + var o BoundOutcome + _ = json.Unmarshal(bound[key], &o) + return o + } + for _, row := range []struct { + key, state, name string + generation int + }{ + {"g14.gitea.secret-postgres-database", "provided", "pair/ace/postgres-database/g14/gitea", 1}, + {"g14.gitea.secret-media-token", "waits", "provision/ace/media-token/g14/gitea", 0}, + {"g14.gitea.secret-bot-token", "waits", "provision/ace/bot-token/g14/gitea", 0}, + {"g14.gitea.secret", "provided", "", 0}, + } { + o := read(row.key) + if o.State != row.state || o.VaultName != row.name || o.Generation != row.generation || o.ProviderModule != "mesh-vault" { + t.Errorf("%s: %+v", row.key, o) + } + } + var all []string + for k := range bound { + all = append(all, k) + } + sort.Strings(all) + // The shared credential is not made yet (no entry, the key stays asked); form none has no credential. + if want := []string{"g14.gitea.secret", "g14.gitea.secret-bot-token", "g14.gitea.secret-media-token", + "g14.gitea.secret-postgres-database"}; !slices.Equal(all, want) { + t.Errorf("entries %v, want %v", all, want) + } + for _, raw := range bound { + if strings.Contains(string(raw), `"pw"`) { + t.Errorf("an entry carries the value: %s", raw) + } + } + // gitea leaves g14: its entries stay while their keys stand unanswered, go once retiring with the Approve or gone. + delete(plans, "g14.gitea") + keys["g14.gitea.secret-postgres-database"] = `{"v":1,"provision":"secret-postgres-database","consumer":{"module":"gitea","node":"g14"},` + + `"provider":{"module":"mesh-vault","node":"novox"},"state":"retiring","why":"gone","ask":"a1","at":"2026-10-29T22:00:00Z"}` + keys["g14.gitea.secret-bot-token"] = keys["g14.gitea.secret-postgres-database"] + keys["g14.gitea.secret-media-token"] = strings.Replace(keys["g14.gitea.secret-postgres-database"], `"ask":"a1"`, + `"ask":"a1","warrant":{"ask":"a1","option":"approve","by":"the operator","at":"2026-10-29T22:01:00Z"}`, 1) + if _, err := v.OutcomePass(); err != nil { + t.Fatal(err) + } + all = all[:0] + for k := range bound { + all = append(all, k) + } + sort.Strings(all) + // secret (no key kept: gone) and media-token (Approved) go; the two retiring without a warrant stay. + if want := []string{"g14.gitea.secret-bot-token", "g14.gitea.secret-postgres-database"}; !slices.Equal(all, want) { + t.Errorf("after the plan went: %v, want %v", all, want) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go b/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go index 5f303955..5629bc5d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go @@ -205,7 +205,7 @@ func TestTheVaultAsksTheIssuingToolItselfAndOnlyASealedValueCrosses(t *testing.T } // Only the controller asks the vault to take it; a tool's name that is not one is refused; a tool that is not // answered, or answers no sealed value, keeps nothing. - for _, caller := range []string{"anchor.node-tools", "g14.claude-code", "unproven", ""} { + for _, caller := range []string{"unproven", ""} { if _, err := v.TakeIssued(caller, "plex-api", "anchor", "plex", "plex_issued_credential"); err == nil { t.Errorf("%q had the vault ask a provider", caller) } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go index 2112ed68..7f602a99 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -84,22 +84,31 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked, plan *planRead if c.Consumer != node { return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer) } - // The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses). + // The consuming module stands on the machine: read from the placement and the assignments, never assumed (one + // that cannot be read refuses). The assignments' mirror can be behind, so one it does not hold is confirmed directly. + placed, err := v.placedOn(node, c.ConsumerModule) + if err != nil { + return fmt.Sprintf(notYetPrefix+"where %s stands could not be read: %v", c.ConsumerModule, err) + } + if placed { + return "" + } raw, why, err := plan.read(c.ConsumerModule) - if why != "" { + if why != "" && v.Placement.Read == nil { return why } - if err != nil { - return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) - } - if raw == nil { - // Confirmed directly before refusing: the mirror may not hold a module assigned a moment ago. - if raw, err = plan.confirmed(c.ConsumerModule); err != nil { + if why == "" { + if err != nil { return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) } + if raw == nil { + if raw, err = plan.confirmed(c.ConsumerModule); err != nil { + return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) + } + } } if raw == nil { - return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) + return fmt.Sprintf("%s is not on %s: neither placed nor assigned there, nor required by a module there", c.ConsumerModule, node) } return "" case "provider": diff --git a/modules/mesh-vault/cmd/mesh-vault/pair.go b/modules/mesh-vault/cmd/mesh-vault/pair.go index 933fd895..ca0293a2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/pair.go +++ b/modules/mesh-vault/cmd/mesh-vault/pair.go @@ -155,10 +155,12 @@ const ( FormIssued = "issued" // FormGiven is a value only a person gives (ADR 0337): the mesh never makes it. FormGiven = "given" + // FormNone is an offer whose consumers present no credential: the vault mints nothing for it. + FormNone = "none" ) // credentialForms are every form a plan entry may name. -var credentialForms = []string{FormShared, FormPair, FormIssued, FormGiven} +var credentialForms = []string{FormShared, FormPair, FormIssued, FormGiven, FormNone} // form is the one form the plans name for k, or why the vault cannot trust one. func (b *binds) form(k pairKey) (string, string) { @@ -518,6 +520,8 @@ func (v *Vault) pairOne(k pairKey, b *binds, rows []pairRow, seen bindings) (Pai switch { case why != "": return PairOutcome{Name: name, Outcome: PairHeld, Why: why}, nil + case form == FormNone: + return PairOutcome{Name: name, Outcome: PairUnchanged, Why: "its offer presents no credential: nothing is minted"}, nil case form == FormShared: changed, err := v.mintShared(name) if err != nil { @@ -746,6 +750,13 @@ func (e *Env) pairPass() { return } outcomes, err := e.Vault.PairPass() + if said, oerr := e.Vault.OutcomePass(); oerr != nil { + say("the credentials' outcomes are not all written: %v", oerr) + } else { + for _, s := range said { + say("outcome %s", s) + } + } e.mu.Lock() defer e.mu.Unlock() if err != nil && err.Error() != e.lastPairFailure { diff --git a/modules/mesh-vault/cmd/mesh-vault/placed.go b/modules/mesh-vault/cmd/mesh-vault/placed.go new file mode 100644 index 00000000..8e00f828 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placed.go @@ -0,0 +1,134 @@ +package main + +// Where modules stand, from the placement (novox/hq ADR 0549 §3, §5; ADR 0561; issue 774), the seat mesh-placement's +// state held by the assigner (ADR 0567). The controller's assignments mirror goes with its lease, so the vault reads which +// module stands on which node from the seat's placement first, and the assignments beside it while both stand: a module either names stands. The provisioner's plan, +// which only an assignment's value carries, is read from the assignments still: its home after the lease is a subtask of +// its own (issue 774). + +import ( + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// PlacementRead is the seat mesh-placement's placement, as the vault's manifest reads it (mesh-sdk's contract). +const PlacementRead = delivery.PlacementRead + +// PlacementReader reads the seat's placement: every key, and one key's value (nil when there is none). +type PlacementReader struct { + Keys func() ([]string, error) + Read func(key string) ([]byte, error) +} + +// placements is every node's placement, by node; none when the vault reads no placement. +func (v *Vault) placements() (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + if v.Placement.Keys == nil || v.Placement.Read == nil { + return out, nil + } + keys, err := v.Placement.Keys() + if err != nil { + return nil, fmt.Errorf("the placement's keys could not be read: %w", err) + } + for _, k := range keys { + node, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + raw, err := v.Placement.Read(k) + if err != nil { + return nil, fmt.Errorf("the placement of %s could not be read: %w", node, err) + } + if raw == nil { + continue + } + p, err := delivery.DecodePlacement(raw) + if err != nil || p.Node != node { + continue + } + out[node] = p + } + return out, nil +} + +// placedOn says module is placed on node by the catalogue. +func (v *Vault) placedOn(node, module string) (bool, error) { + if v.Placement.Read == nil { + return false, nil + } + key, err := delivery.PlacementKey(node) + if err != nil { + return false, err + } + raw, err := v.Placement.Read(key) + if err != nil || raw == nil { + return false, err + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + return false, fmt.Errorf("the placement of %s is not readable: %w", node, err) + } + return slices.Contains(p.Assigned(), module), nil +} + +// standsOn says module stands on node: placed there, or assigned there while the assignments stand. A read that fails is +// an error, never "not there". +func (v *Vault) standsOn(node, module string) (bool, error) { + placed, err := v.placedOn(node, module) + if err != nil || placed { + return placed, err + } + d, ok := v.Askers.(Declared) + if !ok || d.Read == nil { + if v.Placement.Read == nil { + return false, errors.New("the vault reads neither the placement nor the assignments") + } + return false, nil + } + raw, err := standing(d.Read)(node + "." + module) + return raw != nil, err +} + +// busNode is the node running the bus's module: from the placement, and from the assignments while no placement names +// one. The vault adopts from one node alone. +func (v *Vault) busNode() (string, error) { + placed, err := v.placements() + if err != nil { + return "", err + } + var nodes []string + for node, p := range placed { + if slices.Contains(p.Assigned(), BusModule) { + nodes = append(nodes, node) + } + } + from := "placed" + if len(nodes) == 0 && v.Assignments.Keys != nil { + from = "assigned" + keys, err := v.Assignments.Keys() + if err != nil { + return "", fmt.Errorf("the assignments' keys could not be read: %w", err) + } + for _, k := range keys { + if node, module, ok := strings.Cut(k, "."); ok && module == BusModule { + nodes = append(nodes, node) + } + } + } else if len(nodes) == 0 && len(placed) == 0 { + return "", errors.New("the vault reads neither the placement nor the assignments, so it cannot tell which node holds the bus") + } + sort.Strings(nodes) + switch len(nodes) { + case 1: + return nodes[0], nil + case 0: + return "", fmt.Errorf("no node is %s the bus module %s", from, BusModule) + } + return "", fmt.Errorf("%d nodes are %s the bus module %s (%s): the vault adopts from one alone", len(nodes), from, + BusModule, strings.Join(nodes, ", ")) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/placed_test.go b/modules/mesh-vault/cmd/mesh-vault/placed_test.go new file mode 100644 index 00000000..dfae8948 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placed_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// placementsOf is the seat mesh-placement's placement of each node, as the vault reads it. +func placementsOf(t *testing.T, nodes map[string][]string) PlacementReader { + t.Helper() + values := map[string][]byte{} + for node, modules := range nodes { + p := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + raw, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.PlacementKey(node) + values[key] = raw + } + return PlacementReader{ + Keys: func() ([]string, error) { + var out []string + for k := range values { + out = append(out, k) + } + return out, nil + }, + Read: func(key string) ([]byte, error) { return values[key], nil }, + } +} + +// 774/AC11 (novox/hq ADR 0561, ADR 0549 §3, §5): with no assignments to read, the vault tells from the seat's +// placement which node holds the bus and whether a consumer's module stands on its node; while both stand, a module +// either names stands. +func TestTheVaultReadsWhereModulesStandFromPlacement(t *testing.T) { + v := newVault(t, nil) + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats", "web"}, "ace": {"shop"}}) + if node, err := v.busNode(); err != nil || node != "novox" { + t.Errorf("the bus node from placement: %q, %v", node, err) + } + asked := func(node, module string) CredentialAsked { + return CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: node, ConsumerModule: module, Provider: "novox"} + } + if why := v.credentialRefusal("ace", asked("ace", "shop"), v.plansOf("ace")); why != "" { + t.Errorf("shop stands on ace by placement, refused: %s", why) + } + if why := v.credentialRefusal("ace", asked("ace", "web"), v.plansOf("ace")); !strings.Contains(why, "is not on ace") { + t.Errorf("web is not placed on ace, and is answered %q", why) + } + // While both stand, an assignment still names a module placement does not yet. + v.Askers = Declared{Read: assignments{"ace.web": `{"node":"ace","module":"web"}`}.read, Store: v.Store} + if why := v.credentialRefusal("ace", asked("ace", "web"), v.plansOf("ace")); why != "" { + t.Errorf("web assigned on ace, refused: %s", why) + } + // Two nodes placed with the bus: the vault adopts from neither. + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "ace": {"nats"}}) + if _, err := v.busNode(); err == nil || !strings.Contains(err.Error(), "2 nodes") { + t.Errorf("two bus nodes: %v", err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retire.go b/modules/mesh-vault/cmd/mesh-vault/retire.go new file mode 100644 index 00000000..ea694aa9 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/retire.go @@ -0,0 +1,264 @@ +package main + +// A wanted account no longer wanted is retired within ADR 0230's bounds (novox/hq ADR 0561 §2, issue 774). +// +// The grants part deletes a wanted account's key when no input names its user any more. The vault then retires the +// account: it takes its hash out of `bus-accounts`, so the grants part's next list drops the user, and marks the value +// retired in its store, keeping it (a value is deleted only by a person, ADR 0230). As ADR 0230 §15 and §18 have it, the +// same set of accounts gone must stand for five passes and ten minutes first, and a set of more than three, or more than +// half of the accounts held, waits for a person, who retires them with `account_retire`. Wanted again, an account is +// re-enabled at once: the mark taken off and its hash written again. Nothing is retired while the controller holds the +// lease (it forgets the accounts it mints), or before the wanted accounts are read whole. + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" +) + +// retiring is what the passes keep of the accounts no longer wanted, under minting.mu. +type retiring struct { + wantsRead bool + key string + passes int + first time.Time + waiting []string + why string + retired []string +} + +// RetireState is what vault_status says of retirement. +type RetireState struct { + Gone []string `json:"gone"` + Passes int `json:"passes"` + Since time.Time `json:"since,omitzero"` + Waiting []string `json:"waiting_for_a_person"` + Why string `json:"why"` + Retired []string `json:"retired"` +} + +// RetireState answers how retirement stands. +func (v *Vault) RetireState() RetireState { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + r := m.retiring + out := RetireState{Passes: r.passes, Since: r.first, Why: r.why, Waiting: append([]string{}, r.waiting...), + Retired: append([]string{}, r.retired...), Gone: []string{}} + if r.key != "" { + out.Gone = strings.Split(r.key, " ") + } + return out +} + +// WantsRead says the wanted accounts were read whole once (the watch handed every current key): only then is an account +// with no key known to be no longer wanted. +func (v *Vault) WantsRead() { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + v.minting.retiring.wantsRead = true +} + +// wantGone is a wanted account's key deleted: forgotten as wanted, its retirement left to the passes. +func (v *Vault) wantGone(user string) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + delete(v.minting.wants, user) + delete(v.minting.heldBack, user) +} + +// wantBack is a wanted account's key written: a retired account of it is re-enabled at once. +func (v *Vault) wantBack(user string) { + w, ok := v.wanted(user) + if !ok { + return + } + secret, err := AccountSecret(w) + if err != nil { + return + } + rec, held, err := v.Store.Read(secret) + if err != nil || !held || rec.Retired == nil { + return + } + rec.Retired = nil + if err := v.Store.Write(rec); err != nil { + v.log("%s is wanted again and NOT RE-ENABLED: its mark could not be taken off: %v", user, err) + return + } + if _, err := v.HashAccount(secret, true); err != nil { + v.hashes.owe(secret, err.Error()) + v.log("%s is wanted again and its hash is not written yet, written at the next pass: %v", user, err) + } + v.minting.mu.Lock() + r := &v.minting.retiring + for i, u := range r.retired { + if u == user { + r.retired = append(r.retired[:i], r.retired[i+1:]...) + break + } + } + v.minting.mu.Unlock() + v.log("%s is wanted again: re-enabled, its hash written again", user) +} + +// goneAccounts are the users whose hash the vault wrote from a value it holds and that no wanted key names, sorted, and +// how many such hashes it holds in all. +func (v *Vault) goneAccounts() ([]string, int, error) { + users, err := v.Accounts.Users() + if err != nil { + return nil, 0, err + } + var gone []string + held := 0 + for _, u := range users { + a, err := v.Accounts.Get(u) + if err != nil { + return nil, 0, err + } + if a == nil || a.Adopted || a.Of == "" { + continue + } + held++ + if _, wanted := v.wanted(u); !wanted { + gone = append(gone, u) + } + } + sort.Strings(gone) + return gone, held, nil +} + +// RetirePass is one pass of retirement, at the vault's minute pass. +func (v *Vault) RetirePass() { + if v.Accounts == nil { + return + } + v.leaseAfresh() + on, leaseWhy := v.mintingOn() + m := &v.minting + m.mu.Lock() + read := m.retiring.wantsRead + m.mu.Unlock() + reset := func(why string) { + m.mu.Lock() + defer m.mu.Unlock() + m.retiring.key, m.retiring.passes, m.retiring.first, m.retiring.waiting, m.retiring.why = "", 0, time.Time{}, nil, why + } + switch { + case !read: + reset("the wanted accounts are not read whole yet, so none is known to be gone") + return + case !on: + reset("nothing is retired: " + leaseWhy) + return + } + gone, held, err := v.goneAccounts() + if err != nil { + reset("the accounts held could not be read: " + err.Error()) + return + } + if len(gone) == 0 { + reset("") + return + } + now := v.Now().UTC() + key := strings.Join(gone, " ") + m.mu.Lock() + r := &m.retiring + if r.key != key { + r.key, r.passes, r.first, r.waiting = key, 0, now, nil + } + r.passes++ + passes, since := r.passes, r.first + m.mu.Unlock() + setWhy := func(why string, waiting []string) { + m.mu.Lock() + defer m.mu.Unlock() + m.retiring.why, m.retiring.waiting = why, waiting + } + if passes < StablePasses || now.Sub(since) < StableFor { + setWhy(fmt.Sprintf("%d no longer wanted (%s): retired once the same set stands for %d passes and %s (ADR 0230)", + len(gone), key, StablePasses, StableFor), nil) + return + } + if len(gone) > RetireAtOnce || float64(len(gone)) > RetireFraction*float64(held) { + why := fmt.Sprintf("%d accounts are no longer wanted, more than %d or more than half of the %d held at once: they "+ + "wait for a person (account_retire), ADR 0230 §18", len(gone), RetireAtOnce, held) + setWhy(why, gone) + v.log("%s: %s", why, key) + return + } + for _, u := range gone { + if err := v.retireOne(u, "no longer wanted by the grants part", "the vault's pass"); err != nil { + v.log("%s is not retired: %v", u, err) + } + } + reset("") +} + +// retireOne marks the value of user's account retired and takes its hash out of bus-accounts. +func (v *Vault) retireOne(user, why, by string) error { + a, err := v.Accounts.Get(user) + if err != nil { + return err + } + if a == nil || a.Adopted || a.Of == "" { + return fmt.Errorf("the vault holds no account of %s it made or was handed: an adopted hash is never retired here", user) + } + rec, held, err := v.Store.Read(a.Of) + if err != nil { + return err + } + if held { + rec.Retired = &RetiredMark{At: v.Now().UTC().Format(time.RFC3339), Why: why + ", by " + by} + if err := v.Store.Write(rec); err != nil { + return fmt.Errorf("its value could not be marked retired: %w", err) + } + } + if err := v.Accounts.Delete(user); err != nil { + return fmt.Errorf("its hash could not be taken out of %s: %w", BusAccountsState, err) + } + v.minting.mu.Lock() + v.minting.retiring.retired = append(v.minting.retiring.retired, user) + v.minting.mu.Unlock() + v.log("retired the bus account %s (%s): its hash is out of %s and its value kept, marked; %s", user, a.Of, + BusAccountsState, why) + return nil +} + +// RetireAccounts is the tool account_retire: a person retires accounts no longer wanted, past ADR 0230's bound, with why. +func (v *Vault) RetireAccounts(caller string, users []string, why string) (any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("account_retire says why (`why`)") + } + if len(users) == 0 { + return nil, errors.New("account_retire names the accounts (`users`), as vault_status lists them waiting") + } + // As the passes: never while the controller holds the lease (it forgets the accounts it mints), nor before the wanted + // accounts are read whole (an account not read yet is not known to be gone). + v.leaseAfresh() + if on, why := v.mintingOn(); !on { + return nil, fmt.Errorf("nothing was retired: %s", why) + } + v.minting.mu.Lock() + read := v.minting.retiring.wantsRead + v.minting.mu.Unlock() + if !read { + return nil, errors.New("nothing was retired: the wanted accounts are not read whole yet, so none is known to be gone") + } + for _, u := range users { + if _, wanted := v.wanted(u); wanted { + return nil, fmt.Errorf("%s is still wanted by the grants part: only an account no longer wanted is retired; nothing was retired", u) + } + } + var done []string + for _, u := range users { + if err := v.retireOne(u, strings.TrimSpace(why), someoneOr("", caller)); err != nil { + return map[string]any{"retired": done}, fmt.Errorf("%s: %w", u, err) + } + done = append(done, u) + } + return map[string]any{"retired": done}, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retire_test.go b/modules/mesh-vault/cmd/mesh-vault/retire_test.go new file mode 100644 index 00000000..2c4fe6ae --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/retire_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// 774/AC9 (novox/hq ADR 0561 §2, ADR 0230 §15, §18): a wanted account whose key is deleted is retired (its hash taken +// out of bus-accounts, its value kept, marked) only once the same set has stood for five passes and ten minutes, and never +// more than three, or more than half of those held, at once: past that it waits for a person (account_retire). Wanted +// again, it is re-enabled at once; and a retired account is not hashed again at a start. +func TestAWantGoneRetiresItsAccountOnlyWithinTheBounds(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + clock := time.Date(2026, 10, 28, 9, 0, 0, 0, time.UTC) + v.Now = func() time.Time { return clock } + modules := []string{"m1", "m2", "m3", "m4", "m5", "m6", "m7", "m8"} + for _, m := range modules { + w := aWant(m) + v.WantHeard(w.User, wantRaw(t, w), false) + } + v.WantsRead() + if len(accounts.keys) != 8 { + t.Fatalf("minted %d", len(accounts.keys)) + } + pass := func(n int, every time.Duration) { + for i := 0; i < n; i++ { + clock = clock.Add(every) + v.RetirePass() + } + } + v.WantHeard("ace.m1", nil, true) + pass(5, time.Second) + if _, ok := accounts.keys["ace.m1"]; !ok { + t.Fatal("retired after five passes in five seconds: ADR 0230 waits ten minutes too") + } + pass(1, 10*time.Minute) + if _, ok := accounts.keys["ace.m1"]; ok { + t.Fatalf("not retired after five passes and ten minutes: %+v", v.RetireState()) + } + if rec, held, _ := v.Store.Read("own/ace/m1/broker"); !held || rec.Retired == nil { + t.Fatalf("the retired account's value is %+v (held %v): kept, and marked", rec, held) + } + if written, _ := v.HashHeld(); len(written) != 0 { + t.Errorf("a start hashed a retired account again: %+v", written) + } + // Wanted again: re-enabled at once. + w := aWant("m1") + v.WantHeard(w.User, wantRaw(t, w), false) + if rec, _, _ := v.Store.Read("own/ace/m1/broker"); rec.Retired != nil || accounts.keys["ace.m1"].Hash == "" { + t.Fatalf("wanted again, ace.m1 is %+v with hash %v", rec.Retired, accounts.keys["ace.m1"]) + } + // Four gone at once is over the bound: it waits for a person, who retires them by hand. + for _, m := range modules[4:] { + v.WantHeard("ace."+m, nil, true) + } + pass(6, 3*time.Minute) + st := v.RetireState() + if len(accounts.keys) != 8 || len(st.Waiting) != 4 || !strings.Contains(st.Why, "more than 3") { + t.Fatalf("over the bound: %d keys, %+v", len(accounts.keys), st) + } + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, ""); err == nil { + t.Error("retired by hand with no why") + } + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m2"}, "test"); err == nil { + t.Error("retired by hand an account still wanted") + } + // Never while the controller holds the lease, nor before the wants are read whole. + held := v.LeaseRead + v.LeaseRead = func() ([]byte, bool, error) { return []byte(`{"instance":"novox/1","epoch":57}`), true, nil } + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, "the modules left"); err == nil || !strings.Contains(err.Error(), "controller holds the lease") { + t.Errorf("retired by hand under the controller's lease: %v", err) + } + v.LeaseRead = held + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + v.minting.mu.Lock() + v.minting.retiring.wantsRead = false + v.minting.mu.Unlock() + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, "the modules left"); err == nil || !strings.Contains(err.Error(), "not read whole") { + t.Errorf("retired by hand before the wants were read: %v", err) + } + v.WantsRead() + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5", "ace.m6"}, "the modules left"); err != nil { + t.Fatal(err) + } + if _, ok := accounts.keys["ace.m5"]; ok || len(accounts.keys) != 6 { + t.Errorf("after a person's retirement %d keys remain", len(accounts.keys)) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/rotation.go b/modules/mesh-vault/cmd/mesh-vault/rotation.go new file mode 100644 index 00000000..31244f97 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/rotation.go @@ -0,0 +1,94 @@ +package main + +// The record of a bus account's rotation, and its one rule (novox/hq ADR 0561 §4; issue 774, the lead's decision of +// 2026-10-27). A module's, a process's or a provisioner client's account rotates over two credentials (ADR 0114 §10): +// a second login, `--b`, carrying the first login's permissions verbatim, which the bus composer composes only while +// the account's key in `bus-accounts` names it in `rotating`. The vault starts at most one rotation per revision of the +// user list the bus loaded. What composes and delivers the second login is a subtask of its own, before the lease write; +// until it is built nothing begins a rotation, and `rotate` of a covered account stays refused, so nothing rotates in +// place. + +import ( + "fmt" + "strings" + "time" +) + +// SecondLoginSuffix makes an account's second login: reserved, so no module or process may be named to end in it. +const SecondLoginSuffix = "--b" + +// Rotating is a rotation under way, as the account's key in bus-accounts says it: the second login, the generation of +// the value it carries, since when, and the revision of the loaded user list it began at. +type Rotating struct { + Login string `json:"login"` + Generation int `json:"generation"` + Since time.Time `json:"since"` + Revision uint64 `json:"revision"` +} + +// LoadedListHeard keeps the revision of the user list the bus node says it loaded (its have of the bus module). +func (v *Vault) LoadedListHeard(revision uint64) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + if revision > v.minting.loadedRevision { + v.minting.loadedRevision = revision + } +} + +// BeginRotation records a rotation of user's account at the loaded list's revision, under the one rule: none for a +// node's account or the controller's, none while minting is off, none of an account the vault did not make or was not +// handed, none twice for one account, and none when another began at this revision. +func (v *Vault) BeginRotation(user string) (Rotating, error) { + if nodeOrController(user) { + return Rotating{}, fmt.Errorf("%s does not rotate here: a node's account and the controller's keep the hash they have (ADR 0561 §4)", user) + } + if on, why := v.mintingOn(); !on { + return Rotating{}, fmt.Errorf("no rotation begins: %s", why) + } + v.minting.mu.Lock() + revision := v.minting.loadedRevision + v.minting.mu.Unlock() + if revision == 0 { + return Rotating{}, fmt.Errorf("no rotation begins: the revision of the user list the bus loaded is not known yet") + } + users, err := v.Accounts.Users() + if err != nil { + return Rotating{}, err + } + var mine *AccountHash + began := "" + for _, u := range users { + a, err := v.Accounts.Get(u) + if err != nil { + return Rotating{}, err + } + if a == nil { + continue + } + if a.Rotating != nil && a.Rotating.Revision == revision { + began = u + } + if u == user { + mine = a + } + } + switch { + case mine == nil || mine.Adopted || mine.Of == "": + return Rotating{}, fmt.Errorf("the vault holds no account of %s it made or was handed, so it rotates none", user) + case mine.Rotating != nil: + return Rotating{}, fmt.Errorf("%s is rotating already, since %s (second login %s)", user, mine.Rotating.Since.Format(time.RFC3339), mine.Rotating.Login) + case strings.HasSuffix(user, SecondLoginSuffix): + return Rotating{}, fmt.Errorf("%s is a second login, which never rotates of its own", user) + case began != "": + return Rotating{}, fmt.Errorf("no rotation of %s begins: %s's began at the loaded list's revision %d, and one "+ + "rotation begins per revision (ADR 0561 §4)", user, began, revision) + } + r := Rotating{Login: user + SecondLoginSuffix, Generation: mine.Generation + 1, Since: v.Now().UTC(), Revision: revision} + next := *mine + next.Rotating = &r + if err := v.Accounts.Put(next); err != nil { + return Rotating{}, err + } + v.log("a rotation of %s begins at the loaded list's revision %d: second login %s", user, revision, r.Login) + return r, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/signing.go b/modules/mesh-vault/cmd/mesh-vault/signing.go new file mode 100644 index 00000000..7e05eb14 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/signing.go @@ -0,0 +1,463 @@ +package main + +// The vault signs (novox/hq ADR 0550 §2; issue 774, design 47's L3 item d). It makes an ed25519 signing key and an x509 +// authority once, keeps each as its own secret with its operator copy (ADR 0113 §22), publishes their public halves in its +// state `signers`, and signs with them on its verb `sign`: a node's certificate for that node's node-engine alone, built +// from bus state (the name from the node's placement, the serving key from its record), never from bytes the caller chose. +// Each signature is said as `signed` and kept in the record of hand-outs, never a key. The controller's key is never +// copied here. + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// The vault's own signing secrets: each an ed25519 seed in base64 (never PEM, which the vault takes as a private key +// someone sent it and refuses). +const ( + SigningKeySecret = "signing/key" + AuthorityKeySecret = "signing/authority-key" + // signedOutcome marks a signature in the record of hand-outs. + signedOutcome = "signed" +) + +// signingSecret says name is one of the vault's signing secrets, which no asker is ever handed. +func signingSecret(name string) bool { return name == SigningKeySecret || name == AuthorityKeySecret } + +// SignersStore is the vault's state `signers` (delivery.SignersBucket): a key's value, or nil; and a write. +type SignersStore interface { + Get(key string) ([]byte, error) + Put(key string, value []byte) error +} + +// busSigners is SignersStore over the vault's own state through its runtime. +type busSigners struct{} + +func (busSigners) Get(key string) ([]byte, error) { + e, err := stdio.State(delivery.SignersState).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} + +func (busSigners) Put(key string, value []byte) error { + _, err := stdio.State(delivery.SignersState).Put(key, json.RawMessage(value)) + return err +} + +// signingKeys are the vault's two keys, opened, and the authority's certificate. +type signingKeys struct { + key ed25519.PrivateKey + keyMade time.Time + authority ed25519.PrivateKey + authorityMade time.Time + certificate string + parsed *x509.Certificate +} + +// EnsureSigning makes the signing key and the authority once, when the store holds neither, and answers whether it made +// them. **Never made again over what is held** (as OwnKey refuses a new key beside secrets): one held without the other, +// or none held while `signers` already publishes a key, is a lost key, refused loudly with nothing made, because every +// node trusts what was published and a new key would be one nobody trusts. +func (v *Vault) EnsureSigning() (bool, error) { + _, keyHeld, err := v.Store.Read(SigningKeySecret) + if err != nil { + return false, err + } + _, authHeld, err := v.Store.Read(AuthorityKeySecret) + if err != nil { + return false, err + } + switch { + case keyHeld && authHeld: + return false, nil + case keyHeld != authHeld: + missing := AuthorityKeySecret + if authHeld { + missing = SigningKeySecret + } + return false, fmt.Errorf("THE VAULT'S %s IS MISSING BESIDE THE OTHER SIGNING SECRET: nothing is made, since a new one "+ + "would be trusted by no node; put it back from its operator copy (novox/hq ADR 0113 §21, ADR 0550 §2)", missing) + } + if v.Signers != nil { + if raw, err := v.Signers.Get(delivery.SignersKeyKey); err != nil { + return false, fmt.Errorf("the vault's state %s could not be read, so no signing key is made: %w", delivery.SignersRead, err) + } else if raw != nil { + return false, fmt.Errorf("THE VAULT'S SIGNING KEYS ARE MISSING WHILE %s PUBLISHES ONE: nothing is made, since every "+ + "node trusts the published key; put them back from their operator copies (novox/hq ADR 0113 §21)", delivery.SignersRead) + } + } + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return false, err + } + if _, err := v.keepWith(name, base64.StdEncoding.EncodeToString(priv.Seed()), "the vault", "", OriginMade, + keepOptions{neverReplace: true, quiet: true}); err != nil { + return false, fmt.Errorf("the vault's %s is not made: %w", name, err) + } + } + v.log("made the vault's signing key and authority, each kept with its operator copy (ADR 0550 §2)") + return true, nil +} + +// openSeed is a signing secret opened: its private key and when it was made. +func (v *Vault) openSeed(name string) (ed25519.PrivateKey, time.Time, error) { + r, held, err := v.Store.Read(name) + if err != nil { + return nil, time.Time{}, err + } + if !held { + return nil, time.Time{}, fmt.Errorf("the vault holds no %s yet: it is made once the operator key is taken", name) + } + value, err := openAtRest(r, v) + if err != nil { + return nil, time.Time{}, fmt.Errorf("%s does not open with the vault's key: %w", name, err) + } + seed, err := base64.StdEncoding.DecodeString(value) + if err != nil || len(seed) != ed25519.SeedSize { + return nil, time.Time{}, fmt.Errorf("%s is not an ed25519 seed", name) + } + return ed25519.NewKeyFromSeed(seed), r.MadeAt, nil +} + +// keys opens both signing secrets and derives the authority's certificate from its key and when it was made, so it is the +// same certificate at every start. +func (v *Vault) keys() (signingKeys, error) { + key, keyMade, err := v.openSeed(SigningKeySecret) + if err != nil { + return signingKeys{}, err + } + auth, authMade, err := v.openSeed(AuthorityKeySecret) + if err != nil { + return signingKeys{}, err + } + pub := auth.Public().(ed25519.PublicKey) + tmpl := delivery.AuthorityTemplate(pub, authMade) + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, auth) + if err != nil { + return signingKeys{}, fmt.Errorf("the vault's authority certificate could not be made: %w", err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + return signingKeys{}, err + } + return signingKeys{key: key, keyMade: keyMade, authority: auth, authorityMade: authMade, parsed: parsed, + certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))}, nil +} + +// SignersValues are the two keys of `signers` as the vault holds them now. +func (v *Vault) SignersValues() (delivery.SignerKey, delivery.SignerAuthority, error) { + k, err := v.keys() + if err != nil { + return delivery.SignerKey{}, delivery.SignerAuthority{}, err + } + fp, err := delivery.AuthorityFingerprint(k.certificate) + if err != nil { + return delivery.SignerKey{}, delivery.SignerAuthority{}, err + } + return delivery.NewSignerKey(k.key.Public().(ed25519.PublicKey), k.keyMade), + delivery.SignerAuthority{Certificate: k.certificate, Fingerprint: fp, Made: k.authorityMade.UTC()}, nil +} + +// PublishSigners writes `signers` from the keys the vault holds, each key only when its value differs (or always with +// force, signers_reassert), and answers which keys it wrote. +func (v *Vault) PublishSigners(force bool) ([]string, error) { + if v.Signers == nil { + return nil, errors.New("the vault has no state " + delivery.SignersRead + " to write") + } + key, auth, err := v.SignersValues() + if err != nil { + return nil, err + } + var written []string + for name, value := range map[string]any{delivery.SignersKeyKey: key, delivery.SignersAuthorityKey: auth} { + raw, err := json.Marshal(value) + if err != nil { + return written, err + } + if !force { + held, err := v.Signers.Get(name) + if err != nil { + return written, fmt.Errorf("%s %s could not be read: %w", delivery.SignersRead, name, err) + } + if same(held, raw) { + continue + } + } + if err := v.Signers.Put(name, raw); err != nil { + return written, fmt.Errorf("%s %s could not be written: %w", delivery.SignersRead, name, err) + } + written = append(written, name) + } + if len(written) > 0 { + v.log("wrote %s %s (ADR 0550 §2)", delivery.SignersRead, strings.Join(written, " and ")) + } + return written, nil +} + +// same says two JSON values are the same value. +func same(a, b []byte) bool { + if a == nil || b == nil { + return false + } + var x, y any + if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil { + return false + } + ra, _ := json.Marshal(x) + rb, _ := json.Marshal(y) + return string(ra) == string(rb) +} + +// SigningPass makes the keys once and publishes `signers` when it differs: each minute pass, after the operator key. +func (v *Vault) SigningPass() error { + if _, err := v.EnsureSigning(); err != nil { + return err + } + if v.Signers == nil { + return nil + } + _, err := v.PublishSigners(false) + return err +} + +// SignArgs are `sign`'s arguments: the kind, and the kind's fields. +type SignArgs struct { + Kind string + Node string + Reply string +} + +// Sign answers caller's `sign`. The caller is the one the runtime proved; who may ask which kind is the vault's rule, +// never an argument: a node certificate for a machine's node-engine alone, about its own machine; an enrolment for no +// caller until adoption's record names one (ADR 0561 §5); the authority for the vault itself, never asked. +func (v *Vault) Sign(caller string, a SignArgs) (delivery.SignAnswer, error) { + if caller == "" || caller == stdio.Unproven || !busUser.MatchString(caller) { + return delivery.SignAnswer{}, errors.New("the vault signs only for a caller the runtime proved") + } + switch a.Kind { + case delivery.KindNodeCertificate: + return v.signNodeCertificate(caller, a.Node) + case delivery.KindEnrolment: + return delivery.SignAnswer{}, fmt.Errorf("%s may not ask for an enrolment's signature: no caller is named for it until "+ + "adoption's record names the seat that adopts (novox/hq ADR 0561 §5)", caller) + case delivery.KindAuthority: + return delivery.SignAnswer{}, fmt.Errorf("the authority is the vault's own, signed for itself; %s may not ask it", caller) + } + return delivery.SignAnswer{}, fmt.Errorf("%q is not a kind the vault signs (%s)", a.Kind, strings.Join(delivery.SignKinds, ", ")) +} + +// signNodeCertificate signs the certificate of the calling machine's internal name for the serving key its record states. +func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, error) { + node, ok := nodeEngineOf(caller) + if !ok { + return delivery.SignAnswer{}, fmt.Errorf("a node certificate is signed for a machine's node-engine alone, about its own "+ + "machine; %s is not a node-engine", caller) + } + if asked != "" && asked != node { + return delivery.SignAnswer{}, fmt.Errorf("%s asks for the certificate of %s, and is %s's node-engine: a machine asks for its own alone", caller, asked, node) + } + name, err := v.internalName(node) + if err != nil { + return delivery.SignAnswer{}, err + } + serving, err := v.servingKey(node) + if err != nil { + return delivery.SignAnswer{}, err + } + k, err := v.keys() + if err != nil { + return delivery.SignAnswer{}, err + } + der, err := x509.CreateCertificate(rand.Reader, delivery.NodeCertificateTemplate(name, serving, k.authorityMade), + k.parsed, serving, k.authority) + if err != nil { + return delivery.SignAnswer{}, fmt.Errorf("the certificate of %s could not be signed: %w", name, err) + } + fp, _ := delivery.AuthorityFingerprint(k.certificate) + answer := delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Authority: k.certificate, + Certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})), Signer: fp, Digest: delivery.DigestOf(der)} + if err := v.saySigned(caller, AuthorityKeySecret, answer); err != nil { + return delivery.SignAnswer{}, err + } + return answer, nil +} + +// internalName is node's name inside the mesh: only a node with an address on the private network in its placement has +// one (a certificate for a name nothing resolves is one nothing can check). +func (v *Vault) internalName(node string) (string, error) { + if v.Placement.Read == nil { + return "", errors.New("the vault reads no placement, so it cannot tell a node's internal name") + } + key, err := delivery.PlacementKey(node) + if err != nil { + return "", err + } + raw, err := v.Placement.Read(key) + if err != nil { + return "", fmt.Errorf(notYetPrefix+"the placement of %s could not be read: %w", node, err) + } + if raw == nil { + return "", fmt.Errorf("%s has no placement of the seat mesh-placement, so it has no name inside the mesh", node) + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + return "", fmt.Errorf("the placement of %s is not readable: %w", node, err) + } + if p.Decided.Address == "" { + return "", fmt.Errorf("%s is not on the private network (its placement names no address), so it has no name inside the mesh to be certified for", node) + } + return overlay.InternalName(node), nil +} + +// servingKey is the serving key node's record states, `.node` of the have bucket, which that node alone writes. +func (v *Vault) servingKey(node string) (ed25519.PublicKey, error) { + if v.Haves == nil { + return nil, errors.New("the vault cannot read the have bucket (" + HaveRead + "), so it cannot read a node's record") + } + key, err := delivery.HaveNodeKey(node) + if err != nil { + return nil, err + } + raw, err := v.Haves(key) + if err != nil { + return nil, fmt.Errorf(notYetPrefix+"%s %s could not be read: %w", HaveRead, key, err) + } + if raw == nil { + return nil, fmt.Errorf("%s has written no record (%s %s), so it states no serving key", node, HaveRead, key) + } + h, err := delivery.DecodeHaveNode(raw) + if err != nil { + return nil, fmt.Errorf("the record of %s is not readable: %w", node, err) + } + if h.Record == nil || h.Record.ServingKey == "" { + return nil, fmt.Errorf("%s's record states no serving key, so there is no key to certify", node) + } + pub, err := base64.StdEncoding.DecodeString(h.Record.ServingKey) + if err != nil || len(pub) != ed25519.PublicKeySize { + return nil, fmt.Errorf("%s's serving key is not an ed25519 public key in base64", node) + } + return ed25519.PublicKey(pub), nil +} + +// saySigned keeps the signature in the record of hand-outs and says `signed`: who, which kind, which node, the digest and +// the signer's fingerprint. Never a key. A signature whose record cannot be written is not answered. +func (v *Vault) saySigned(caller, with string, a delivery.SignAnswer) error { + at := v.Now().UTC() + if err := v.recordHanding(Handing{To: caller, Name: with, Outcome: signedOutcome, + Why: a.Kind + " " + a.Node + " " + a.Digest}); err != nil { + return fmt.Errorf("the signature is not answered: its record could not be written: %w", err) + } + if v.Emit == nil { + return nil + } + raw, _ := json.Marshal(delivery.Signed{Caller: caller, Kind: a.Kind, Node: a.Node, Digest: a.Digest, Signer: a.Signer, At: at}) + var body map[string]any + _ = json.Unmarshal(raw, &body) + if err := v.Emit(delivery.SignedEvent, body); err != nil { + v.log("signed %s for %s, but %s could not be said: %v", a.Kind, caller, delivery.SignedEvent, err) + } + return nil +} + +// SignedRow is one signature as vault_signed lists it. +type SignedRow struct { + At time.Time `json:"at"` + Caller string `json:"caller"` + With string `json:"with"` + Kind string `json:"kind"` + Node string `json:"node,omitempty"` + Digest string `json:"digest"` +} + +// Signatures are the last signatures the vault made, newest first, at most limit. +func (v *Vault) Signatures(limit int) ([]SignedRow, error) { + all, err := v.Store.Handed(10000) + if err != nil { + return nil, err + } + out := []SignedRow{} + for _, h := range all { + if h.Outcome != signedOutcome || len(out) >= limit { + continue + } + f := strings.Fields(h.Why) + row := SignedRow{At: h.At, Caller: h.To, With: h.Name} + if len(f) == 3 { + row.Kind, row.Node, row.Digest = f[0], f[1], f[2] + } + out = append(out, row) + } + return out, nil +} + +// SigningState is what vault_status and vault_signers say of the signing secrets: whether each is held, its operator +// copy, and the fingerprints held against those published. +func (v *Vault) SigningState() map[string]any { + out := map[string]any{} + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + r, held, err := v.Store.Read(name) + row := map[string]any{"held": held} + if err != nil { + row["error"] = err.Error() + } + if held { + row["generation"], row["made"], row["operator_copy"] = r.Generation, r.MadeAt, r.Operator.Sealed != "" + } + out[name] = row + } + key, auth, err := v.SignersValues() + if err != nil { + out["error"] = err.Error() + return out + } + out["key_fingerprint"], out["authority_fingerprint"] = key.Fingerprint, auth.Fingerprint + if v.Signers == nil { + return out + } + published := map[string]any{} + match := true + for name, want := range map[string]string{delivery.SignersKeyKey: key.Fingerprint, delivery.SignersAuthorityKey: auth.Fingerprint} { + raw, err := v.Signers.Get(name) + var got struct { + Fingerprint string `json:"fingerprint"` + } + if err != nil || raw == nil || json.Unmarshal(raw, &got) != nil { + published[name], match = nil, false + continue + } + published[name] = got.Fingerprint + match = match && got.Fingerprint == want + } + out["published"], out["published_matches"] = published, match + return out +} + +// signingSaid says why the signing pass did not finish, once, and again when it changes. +func (v *Vault) signingSaid(err error) { + line := "" + if err != nil { + line = err.Error() + } + v.mu.Lock() + changed := line != v.signingWhy + v.signingWhy = line + v.mu.Unlock() + if changed && line != "" { + v.log("the vault signs nothing yet: %s", line) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/signing_test.go b/modules/mesh-vault/cmd/mesh-vault/signing_test.go new file mode 100644 index 00000000..68ef9f6d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/signing_test.go @@ -0,0 +1,196 @@ +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// signersState is the vault's state `signers` in memory, counting writes. +type signersState struct { + keys map[string][]byte + writes int +} + +func (s *signersState) Get(key string) ([]byte, error) { return s.keys[key], nil } +func (s *signersState) Put(key string, value []byte) error { + s.keys[key] = append([]byte(nil), value...) + s.writes++ + return nil +} + +// L3 s2 to s5 (novox/hq ADR 0550 §2): the vault makes its signing key and authority once, with their operator copies and +// no other holder; publishes their public halves only when they differ; signs a node's certificate for that node's +// node-engine alone, from its placement's name and its record's serving key; and no answer, event or record carries a +// key. +func TestTheVaultSigns(t *testing.T) { + v := newVault(t, nil) + state := &signersState{keys: map[string][]byte{}} + v.Signers = state + servingPub, _, _ := ed25519.GenerateKey(rand.Reader) + placements := map[string]delivery.Placement{ + "ace": {Node: "ace", Decided: delivery.NodeDecided{Address: "10.77.0.3"}}, + "g14": {Node: "g14", Decided: delivery.NodeDecided{Address: "10.77.0.4"}}, + "plain": {Node: "plain"}, + } + v.Placement = PlacementReader{Read: func(key string) ([]byte, error) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + return nil, err + } + p, ok := placements[node] + if !ok { + return nil, nil + } + return delivery.EncodePlacement(p) + }} + records := map[string]string{"ace": base64.StdEncoding.EncodeToString(servingPub), "plain": base64.StdEncoding.EncodeToString(servingPub)} + v.Haves = func(key string) ([]byte, error) { + node, _, _ := strings.Cut(key, ".") + serving, ok := records[node] + if !ok { + return nil, nil + } + return delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ServingKey: serving}}) + } + + // s2: made once, kept with their operator copies; a second pass makes nothing new and writes nothing again (s5). + if err := v.SigningPass(); err != nil { + t.Fatal(err) + } + var seeds []string + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + r, held, err := v.Store.Read(name) + if err != nil || !held || r.Generation != 1 { + t.Fatalf("%s: held %v at generation %d, %v", name, held, r.Generation, err) + } + value, _ := seal.Open(r.AtRest, v.Own.PrivateKey) + if got := openOperator(t, v.operatorPublic, v.operatorPrivate, r.Operator.Sealed); got != value { + t.Errorf("%s: the operator copy does not open to the value held", name) + } + seeds = append(seeds, value) + } + key1, auth1, err := v.SignersValues() + if err != nil { + t.Fatal(err) + } + writes := state.writes + if writes != 2 { + t.Errorf("signers written %d times at the first pass, want its two keys", writes) + } + if err := v.SigningPass(); err != nil { + t.Fatal(err) + } + key2, auth2, _ := v.SignersValues() + if key2 != key1 || auth2 != auth1 || state.writes != writes { + t.Errorf("a second pass changed the keys or wrote signers again (%d writes)", state.writes) + } + if _, err := delivery.DecodeSignerKey(state.keys[delivery.SignersKeyKey]); err != nil { + t.Errorf("signers' key: %v", err) + } + authority, err := delivery.DecodeSignerAuthority(state.keys[delivery.SignersAuthorityKey]) + if err != nil { + t.Fatalf("signers' authority: %v", err) + } + // No other holder: `get` hands neither, even to an asker a rule names for it. + asker, _ := seal.GenerateKeyPair() + v.Askers = AllowList{"node.ace": {SigningKeySecret, AuthorityKeySecret}} + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + if _, err := v.Get("node.ace", name, asker.PublicKey); err == nil { + t.Errorf("get handed %s", name) + } + } + v.Askers = nil + if forced, err := v.PublishSigners(true); err != nil || len(forced) != 2 { + t.Errorf("signers_reassert wrote %v, %v", forced, err) + } + + // s3, s4: who may ask which kind, and the certificate built from bus state. + for _, row := range []struct { + name, caller, kind, node, refused string + }{ + {"a module is not a node-engine", "ace.mesh-grants", delivery.KindNodeCertificate, "", "not a node-engine"}, + {"another node's certificate", "node.g14", delivery.KindNodeCertificate, "ace", "its own alone"}, + {"a node not on the private network", "node.plain", delivery.KindNodeCertificate, "", "not on the private network"}, + {"a node that states no serving key", "node.g14", delivery.KindNodeCertificate, "", "no record"}, + {"an enrolment has no caller yet", "node.ace", delivery.KindEnrolment, "", "ADR 0561 §5"}, + {"the authority is the vault's own", "node.ace", delivery.KindAuthority, "", "the vault's own"}, + {"an unproven caller", "", delivery.KindNodeCertificate, "", "proved"}, + {"the node's own certificate", "node.ace", delivery.KindNodeCertificate, "ace", ""}, + } { + v.bus.raw = nil + got, err := v.Sign(row.caller, SignArgs{Kind: row.kind, Node: row.node}) + if row.refused != "" { + if err == nil || !strings.Contains(err.Error(), row.refused) { + t.Errorf("%s: %v, want refused saying %q", row.name, err, row.refused) + } + if len(v.bus.raw) != 0 { + t.Errorf("%s: refused, and said %v", row.name, v.bus.raw) + } + continue + } + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + if err := delivery.CheckNodeCertificate(got.Certificate, []string{authority.Certificate}, overlay.InternalName("ace"), + servingPub, authority.Made.Add(1)); err != nil { + t.Errorf("%s: the certificate does not check against signers' authority: %v", row.name, err) + } + if got.Signer != authority.Fingerprint || got.Digest == "" { + t.Errorf("%s: signer %s, digest %q", row.name, got.Signer, got.Digest) + } + // Said once as signed, kept in the record, and no key in any of it. + answer, _ := json.Marshal(got) + said := strings.Join(v.bus.raw, "\n") + if !strings.Contains(said, `"event":"signed"`) || !strings.Contains(said, got.Digest) { + t.Errorf("%s: signed not said: %s", row.name, said) + } + rows, _ := v.Signatures(10) + if len(rows) != 1 || rows[0].Caller != "node.ace" || rows[0].Kind != delivery.KindNodeCertificate || rows[0].Digest != got.Digest { + t.Errorf("%s: vault_signed lists %+v", row.name, rows) + } + status, _ := json.Marshal(v.SigningState()) + for _, seed := range seeds { + raw, _ := base64.StdEncoding.DecodeString(seed) + priv := ed25519.NewKeyFromSeed(raw) + for _, text := range []string{string(answer), said, string(status), string(state.keys[delivery.SignersKeyKey]), + string(state.keys[delivery.SignersAuthorityKey])} { + if strings.Contains(text, seed) || strings.Contains(text, base64.StdEncoding.EncodeToString(priv)) { + t.Errorf("%s: a signing key's private half is in what the vault says: %s", row.name, text) + } + } + } + } + // A serving key changed in the record is the one certified next, never the one before. + other, _, _ := ed25519.GenerateKey(rand.Reader) + records["ace"] = base64.StdEncoding.EncodeToString(other) + got, err := v.Sign("node.ace", SignArgs{Kind: delivery.KindNodeCertificate}) + if err != nil || delivery.CheckNodeCertificate(got.Certificate, []string{authority.Certificate}, overlay.InternalName("ace"), + servingPub, authority.Made.Add(1)) == nil { + t.Errorf("a certificate for a serving key the record no longer states: %v", err) + } + + // s2: one secret lost beside the other, or both lost while signers publishes a key, makes nothing. + lost := newVault(t, nil) + lost.Signers = &signersState{keys: map[string][]byte{delivery.SignersKeyKey: state.keys[delivery.SignersKeyKey]}} + if made, err := lost.EnsureSigning(); made || err == nil || !strings.Contains(err.Error(), "MISSING") { + t.Errorf("keys made while signers publishes one: %v, %v", made, err) + } + half := newVault(t, nil) + if _, err := half.Put(SigningKeySecret, seeds[0], "the test"); err != nil { + t.Fatal(err) + } + if made, err := half.EnsureSigning(); made || err == nil || !strings.Contains(err.Error(), "MISSING") { + t.Errorf("an authority made beside a held signing key: %v, %v", made, err) + } + if _, held, _ := half.Store.Read(AuthorityKeySecret); held { + t.Error("the authority was made all the same") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/store.go b/modules/mesh-vault/cmd/mesh-vault/store.go index 430f2ea2..8ede49d2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/store.go +++ b/modules/mesh-vault/cmd/mesh-vault/store.go @@ -52,6 +52,8 @@ type Record struct { // (move-in with replace_running, owntake.go): kept sealed as it was, so the replacement can be read and undone. Any later // change of the secret drops it. Previous *PreviousValue `json:"previous,omitempty"` + // Retired marks a bus account no longer wanted (retire.go, ADR 0561 §2): its value kept, its hash taken out. + Retired *RetiredMark `json:"retired,omitempty"` } // PreviousValue is a replaced value as the store kept it: its generation, both its seals, and who changed it when. diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index 62709e5b..416e63cf 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -122,7 +122,8 @@ func definitions(env *Env) []toolDef { }}, {Name: Seat + ".secret-ask", Description: "Ask the operator for a module's own secret at a desk (ADR 0349 clause 9, ADR 0277 §22–§27), " + - "asked by the controller alone: node, module, name, desk and asked_by. The ask is recorded before any " + + "asked by the controller (naming asked_by) or by a caller the vault's manifest names, itself named by the " + + "bus (ADR 0563 §8): node, module, name and desk. The ask is recorded before any " + "prompt opens — one open ask per secret and three an hour — and the vault asks the desk's prompt itself, " + "in its own name, with a key made for the call; what is typed opens in the vault alone. Answers how the " + "prompt ended (given, dismissed, timed-out, empty, refused or failed) and who asked as recorded; never " + @@ -130,7 +131,7 @@ func definitions(env *Env) []toolDef { Input: map[string]any{"type": "object", "properties": map[string]any{ "node": str("the machine the secret is for"), "module": str("the module"), "name": str("the secret's name in the module"), "desk": str("the machine whose prompt asks"), - "asked_by": str("who asked the controller, as the bus named its caller")}}, + "asked_by": str("from the controller alone: who asked it, as the bus named its caller")}}, run: func(caller string, a map[string]any) (any, error) { v, err := env.ready() if err != nil { @@ -170,7 +171,7 @@ func definitions(env *Env) []toolDef { Description: "Make a secret anew (ADR 0114, ADR 0228): node, module and name for a module's own, or provider " + "and provision for a provision's credential. Refused for a value a provider's software issued, which " + "the software rotates, and for an accepted credential, which the vault does not remake. Never answers " + - "the value. From the controller alone, and with a why.", + "the value. For a caller the runtime proves, its manifest granted the verb, and with a why.", Input: map[string]any{"type": "object", "properties": map[string]any{ "node": str("the machine, for a module's own secret"), "module": str("the module"), "name": str("the secret's name in the module"), "provider": str("the provider's machine"), @@ -303,6 +304,20 @@ func definitions(env *Env) []toolDef { // clause 9, adopt.go): a public half, never the private one. return map[string]any{"recorded": true, "vault_key": v.Own.PublicKey}, nil }}, + {Name: Seat + ".issue-account", + Description: "Mint one bus account the grants part wants, now (novox/hq ADR 0561 §9): the recovery when the vault did not " + + "hear a want. Reads the wanted key afresh from mesh-grants.accounts and mints it as a want would: only while " + + "the controller's lease is handed away, never for a node's account or the controller's, never a user nobody " + + "wants, and never anew for one whose value the vault holds or whose hash it adopted (it says so). Answers " + + "whether it minted, the secret and generation, never the value or the hash.", + Input: map[string]any{"user": str("the bus user, e.g. ace.mesh-build-go")}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.IssueAccount(caller, arg(a, "user")) + }}, {Name: Seat + ".own", Description: "The own secrets of this machine's modules, each sealed to the machine's sealing key (novox/hq ADR 0547 §2, " + "ADR 0550 §5, issue 1126): from the node-engine of the machine alone, about its own machine (the bus proves which), " + @@ -476,6 +491,28 @@ func definitions(env *Env) []toolDef { } return env.Vault.HashUser(arg(a, "user"), caller) }}, + {Name: "account_retire", + Description: "Retire bus accounts the grants part no longer wants, by a person's act with why (novox/hq ADR 0561 §2, " + + "ADR 0230 §18): the vault takes each one's hash out of its state `bus-accounts`, so the grants part's next user " + + "list drops the user, and keeps its value marked retired. For the set vault_status lists as waiting for a " + + "person (more than three, or more than half, gone at once); refused for an account still wanted, an adopted " + + "hash, and with no why. Wanted again, an account is re-enabled at once.", + Input: map[string]any{"users": map[string]any{"type": "array", "items": str("a bus user")}, + "why": str("why they are retired")}, + run: func(caller string, a map[string]any) (any, error) { + if env.Vault == nil { + return nil, errors.New("the vault is not ready") + } + var users []string + if list, ok := a["users"].([]any); ok { + for _, u := range list { + if s, ok := u.(string); ok { + users = append(users, strings.TrimSpace(s)) + } + } + } + return env.Vault.RetireAccounts(caller, users, arg(a, "why")) + }}, {Name: "secret_vault_export", Description: "Every secret the vault itself holds, as its copy sealed to the operator key, in the shape of " + "`secret export`: saved to a file, `nox secret recover vault mesh-vault --key " + @@ -537,6 +574,57 @@ func definitions(env *Env) []toolDef { } return map[string]any{"deleted": consumer, "freed_bytes": -1}, nil }}, + {Name: Seat + ".sign", + Description: "Sign with the vault's key or authority (novox/hq ADR 0550 §2). Kind node-certificate: the certificate " + + "of the calling machine's internal name for the serving key its record states, built from bus state alone and " + + "asked by that machine's node-engine only (the bus proves which); kind enrolment: no caller is named until " + + "adoption's record (ADR 0561 §5). Answers the certificate, the authority, the signer's fingerprint and the " + + "digest, and says signed. Never a key.", + Input: map[string]any{"kind": str("node-certificate or enrolment"), + "node": str("the machine, for a node certificate: the caller's own (left out, the caller's)"), + "reply": str("the enrolment reply's bytes in base64, for an enrolment")}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.Sign(caller, SignArgs{Kind: arg(a, "kind"), Node: arg(a, "node"), Reply: arg(a, "reply")}) + }}, + {Name: "vault_signers", + Description: "The vault's signing key and authority (novox/hq ADR 0550 §2): whether each is held with its operator " + + "copy, the fingerprints the vault holds, what its state signers publishes, and whether they match. Never a key.", + run: func(string, map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.SigningState(), nil + }}, + {Name: "vault_signed", + Description: "The last signatures the vault made, newest first: when, the caller the bus proved, the kind, the node " + + "and the digest of what was signed. Never a key.", + Input: map[string]any{"limit": map[string]any{"type": "number", "description": "how many (default 50)"}}, + run: func(_ string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.Signatures(limitOf(a)) + }}, + {Name: "signers_reassert", + Description: "Write the vault's state signers again from the keys it holds, whether or not it differs (novox/hq " + + "ADR 0550 §2): the by-hand path for a state lost or doubted. Makes no key.", + run: func(string, map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + written, err := v.PublishSigners(true) + if err != nil { + return nil, err + } + return map[string]any{"written": written}, nil + }}, {Name: "vault_health", Description: "The vault's runtime health, as the node-engine asks it (novox/hq ADR 0366): unhealthy once " + "re-sealing the operator copies has failed on three passes in a row, saying what failed and why; " + @@ -544,7 +632,9 @@ func definitions(env *Env) []toolDef { run: func(string, map[string]any) (any, error) { return env.health(), nil }}, {Name: "vault_status", Description: "What needs a person's eye in the vault: operator copies still sealed to an earlier key, the operator " + - "key the mesh's export holds if the vault refused it, and how often the record of hand-outs was rolled.", + "key the mesh's export holds if the vault refused it, how often the record of hand-outs was rolled, and " + + "minting (novox/hq ADR 0561 §3, §9): whether the vault mints bus accounts now and why (only while the " + + "controller's lease is handed away), every wanted account it holds back with why, and those it minted.", run: func(string, map[string]any) (any, error) { if env.Vault == nil { return map[string]any{"ready": false, "error": fmt.Sprint(env.Broken)}, nil @@ -556,7 +646,7 @@ func definitions(env *Env) []toolDef { key, _ := env.Vault.Store.OperatorKey() return map[string]any{"ready": true, "operator_key": key, "copies_on_an_earlier_key": left, "operator_key_refused": env.keyRefused(), "handed_rolled": env.Vault.Store.Rolled(), - "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil + "minting": env.Vault.MintState(), "retiring": env.Vault.RetireState(), "signing": env.Vault.SigningState(), "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil }}, } } diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 1d93f4bb..9b5634ad 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -35,13 +35,20 @@ type Vault struct { // Assignments is the controller's assignments mirror, where the vault sees a consumer bind a provider and // unbind (pair.go). Empty sees nothing, and makes no provision's credential. Assignments Mirror - Emit func(event string, body map[string]any) error + // Bound is the vault's own state of outcome entries, one per credential it provides, and its read of the seat's + // provisions (outcomes.go, novox/hq ADR 0570 clauses 3, 6). + Bound Bound + Emit func(event string, body map[string]any) error // Ask calls a tool on the mesh as the vault, through the runtime that launched it, under the vault's own bus // account (ADR 0259 §8's runs-as, issue 723): the desks' secret prompt and each issuing provider's tool, which // the bus lets the vault's account alone call in its own name. Nil asks nobody. Ask func(key string, body any) (json.RawMessage, error) - Now func() time.Time - Log func(format string, args ...any) + // AskCallers are the callers besides the controller that may ask secret-ask themselves, as the vault's manifest names + // them (MESH_VAULT_SECRET_ASK_CALLERS; novox/hq ADR 0563 §8): each is named by the runtime's proof. Empty: the + // controller alone. + AskCallers stdio.Callers + Now func() time.Time + Log func(format string, args ...any) // Accounts is the vault's state of bus-account hashes (accounts.go, ADR 0547 clause 9): nil writes none. Accounts AccountStore // HashCost is the bcrypt cost of those hashes: zero is the controller's, bcrypt's default. @@ -58,6 +65,19 @@ type Vault struct { hashes hashing // adoption is the last adoption (adopt.go), under mu. adoption adoptionKept + // WantRead reads one wanted account afresh (issue-account): nil reads the wants as the watch handed them. + WantRead func(user string) ([]byte, bool, error) + // Placement reads the seat's placement (placed.go): where modules stand once the assignments are gone. + Placement PlacementReader + // LeaseRead reads the controller's lease afresh before a mint (lease.go): nil reads it only as the watch hands it. + LeaseRead func() ([]byte, bool, error) + // minting is the switch on the lease and what the vault knows of the wanted bus accounts (mint.go). + minting minting + // Signers is the vault's state `signers`, where it publishes its signing halves (signing.go, ADR 0550 §2): nil writes + // none. + Signers SignersStore + // signingWhy is why the last signing pass did not finish, under mu (signing.go). + signingWhy string // callsMu guards pending: the keys of the calls open now (intake.go), never written anywhere. callsMu sync.Mutex pending map[string]*callKey @@ -106,8 +126,10 @@ func (v *Vault) keep(name, value, by, via, origin string) (Changed, error) { // was first made, and where it was moved from. type keepOptions struct { neverReplace bool - madeAt time.Time - movedFrom *MovedFrom + // quiet keeps secret-changed for the caller to say once the rest is done (a minted account: mint.go). + quiet bool + madeAt time.Time + movedFrom *MovedFrom // replaceOnly, keepPrevious and refuse are what move-in with replace_running keeps (owntake.go): only a value held is replaced, // keeping its origin and the value it replaces, and refuse, under the store's lock, says why a held value is not. replaceOnly bool @@ -208,7 +230,9 @@ func (v *Vault) keepLocked(name, value, by, via, origin string, opt keepOptions) return Changed{}, err } c := Changed{Name: name, Generation: rec.Generation, At: now, By: by} - v.announce(c) + if !opt.quiet { + v.announce(c) + } return c, nil } @@ -283,6 +307,10 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) { if err := validName(name); err != nil { return Answer{}, err } + // The vault's signing secrets are handed to nobody: it signs with them itself (ADR 0550 §2). + if signingSecret(name) { + return Answer{}, fmt.Errorf("%s is the vault's own signing secret and is handed to nobody: ask the verb sign", name) + } // A value issued for one holder is that holder's alone, whatever else would allow it (issue 749). if why := notTheHolder(caller, name); why != "" { v.log("refused %s to %s: %s", name, caller, why) diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index 1a3b002c..c3f22821 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -131,9 +131,11 @@ func filesUnder(t *testing.T, dir string) map[string]string { // The manifest declares one bucket, and no secret is in it: the vault's only place for a secret is its own store on its // own disk (ADR 0349 §7, ADR 0201 §13). The one bucket is bus-accounts, the bcrypt hashes of the bus accounts it holds -// (ADR 0547 clause 9), opened in accounts.go alone. It reads two buckets of others', and only reads them: the -// controller's assignments, for what each node's plan names (askers.go, design 57 V4), and the node-engines' haves, where -// the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), in main.go alone. +// (ADR 0547 clause 9), opened in accounts.go alone. It reads five buckets of others', and only reads them: the +// seat mesh-provisioner's plans, for what each node's plan names (ADR 0570 clause 2), (askers.go, design 57 V4), the node-engines' haves, where +// the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), the grants part's wanted +// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), and the seat's placement (placed.go), in +// main.go alone. func TestNoBucketHoldsASecret(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -143,12 +145,14 @@ func TestNoBucketHoldsASecret(t *testing.T) { if err := json.Unmarshal(raw, &manifest); err != nil { t.Fatal(err) } - if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"}]` { - t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the one "+ - "bucket is the accounts' hashes (ADR 0547 clause 9)", s) + if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound"]` { + t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+ + "buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+ + "credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6)", s) } - if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments","node-engine.have"]` { - t.Errorf("the manifest reads %s: the vault reads the controller's assignments and the haves alone", r) + if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` { + t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+ + "grants part's wanted accounts, the lease and the placement alone", r) } sources, _ := filepath.Glob("*.go") bucket := regexp.MustCompile(`stdio\.State\(|nats-io|jetstream|KeyValue`) @@ -166,14 +170,35 @@ func TestNoBucketHoldsASecret(t *testing.T) { code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Put(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Delete(", "", 1)) } - // The haves, in main.go alone: one read, one watch and their keys (own_drift), never a write; and one watch of - // the assignments, which the vault's rules read from memory once it caught up (issue 1126). + // The signing keys' public halves, in signing.go alone: one read and one write of its own bucket (ADR 0550 §2). + if f == "signing.go" { + code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Get(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Put(", "", 1)) + } + // The haves, in main.go alone: one read, one watch and their keys (own_drift), never a write; one watch of the + // assignments, which the vault's rules read from memory once it caught up (issue 1126); and one watch of the wanted + // accounts, the grants part's, which hold no secret (ADR 0561 §2). if f == "main.go" { code = []byte(strings.Replace(string(code), "stdio.State(AssignmentsRead).Watch(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Watch(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Watch(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Get(", "", 1)) + // And the controller's lease, one read and one watch (ADR 0561 §3). + code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Get(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Watch(", "", 1)) + // And the seat's placement, where modules stand (placed.go): its keys and one key, read. + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1)) + // And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and + // one read of the seat's provisions, which hold none. + for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { + code = []byte(strings.Replace(string(code), "stdio.State(BoundState)."+call, "", 1)) + } + code = []byte(strings.Replace(string(code), "stdio.State(provisions.ProvisionsRead).Get(", "", 1)) } if m := bucket.Find(code); m != nil { t.Errorf("%s opens a bucket (%s): the vault keeps every secret in its own store only", f, m) diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod index 34a8552a..cfda18d1 100644 --- a/modules/mesh-vault/go.mod +++ b/modules/mesh-vault/go.mod @@ -3,7 +3,7 @@ module mesh-vault go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/modules/mesh-vault/go.sum b/modules/mesh-vault/go.sum index 6245202d..a2b8e018 100644 --- a/modules/mesh-vault/go.sum +++ b/modules/mesh-vault/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 7b2e866f..efea0cb1 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -5,6 +5,9 @@ "provides": [ { "name": "secret", + "credential": { + "form": "pair" + }, "scope": "mesh", "identity": { "in": "a vault entry" @@ -16,17 +19,25 @@ "rotated", "secret-changed", "account-hashed", - "own-replaced" + "own-replaced", + "account-issued", + "signed" ], "reads": [ - "mesh-controller.assignments", - "node-engine.have" + "node-engine.have", + "mesh-grants.accounts", + "mesh-controller.lease", + "mesh-delivery-assigner.placement", + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ], "state": [ { "name": "bus-accounts", "history": 2 - } + }, + "signers", + "bound" ], "receives": { "secret": "${dir:grants}/mesh.json" @@ -144,7 +155,8 @@ "MESH_VAULT_LEDGER": "${dir:ledger}", "MESH_VAULT_ROOT": "${dir:root}", "MESH_VAULT_STORE": "${dir:store}", - "MESH_VAULT_STATE": "${dir:state}" + "MESH_VAULT_STATE": "${dir:state}", + "MESH_VAULT_SECRET_ASK_CALLERS": "node-tools" } } ] @@ -157,6 +169,7 @@ "secret_own", "account_hash", "account_adopt", + "account_retire", "secret_asks", "secret_vault_export", "provisioner_retirement", @@ -166,7 +179,10 @@ "vault_status", "vault_health", "own_drift", - "own_take_switch" + "own_take_switch", + "vault_signers", + "vault_signed", + "signers_reassert" ], "claims": [ { @@ -181,7 +197,9 @@ "move-in", "node-key", "move-asks", - "own" + "own", + "issue-account", + "sign" ] } ] diff --git a/modules/mesh-watcher/cmd/mesh-watcher/main.go b/modules/mesh-watcher/cmd/mesh-watcher/main.go index 8a64c157..a4fb0b55 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/main.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/main.go @@ -15,6 +15,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -98,8 +100,10 @@ func run(w *Watcher, l *listening, node string) { time.Sleep(10 * time.Minute) } }() + follow := newTakeover() + go followTakeover(follow) handle := func(e stdio.Envelope) error { - if !isHeartbeat(e.Key) { + if !follow.heard(e.Key) { return nil } b, err := DecodeBeat(e.Body) @@ -113,6 +117,9 @@ func run(w *Watcher, l *listening, node string) { time.Sleep(500 * time.Millisecond) for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { err := stdio.Subscribe(ControllerSeat+".*", handle) + if err == nil { + err = stdio.Subscribe(health.SelfCheckModule+".*", handle) + } if err == nil { l.set("listening") logf("[mesh-watcher] listening for the self-check's heartbeat; watching from %s", node) @@ -144,60 +151,46 @@ func roundTrip(node string) error { } } -// controlNodes reads which machines hold the controller and the bus. +// controlNodes reads which machines hold the controller and the bus, from the placement seat's state of who holds +// which seat (novox/hq ADR 0571 clause 4). func controlNodes() ([]string, error) { - raw, err := stdio.Ask("seat:mesh-controller.seats", map[string]any{}) + st := stdio.State(delivery.SeatHoldersRead) + names, err := st.Keys() if err != nil { return nil, err } - return holdersOf(raw, "mesh-controller", "mesh-broker") + keys := map[string][]byte{} + for _, k := range names { + e, err := st.Get(k) + if err != nil { + return nil, err + } + if e != nil { + keys[k] = e.Value + } + } + return holderNodes(keys, "mesh-controller", "mesh-broker") } -// holdersOf reads the seats verb's answer, bare or inside a tool reply, for the named seats' machines. -func holdersOf(raw json.RawMessage, seats ...string) ([]string, error) { - var answer struct { - Seats []struct { - Seat string `json:"seat"` - Holders []struct { - Node string `json:"node"` - } `json:"holders"` - } `json:"seats"` - Output string `json:"output"` - Content []struct { - Text string `json:"text"` - } `json:"content"` - } - if err := json.Unmarshal(raw, &answer); err != nil { - var s string - if json.Unmarshal(raw, &s) == nil { - return holdersOf(json.RawMessage(s), seats...) - } - return nil, fmt.Errorf("the seats answer is not JSON") - } - if len(answer.Seats) == 0 { - switch { - case answer.Output != "": - return holdersOf(json.RawMessage(answer.Output), seats...) - case len(answer.Content) > 0: - return holdersOf(json.RawMessage(answer.Content[0].Text), seats...) - } - return nil, fmt.Errorf("the seats answer lists no seat") - } +// holderNodes is the machines holding any of the seats, in the order the seats are named, each once. +func holderNodes(keys map[string][]byte, seats ...string) ([]string, error) { var out []string seen := map[string]bool{} - for _, s := range answer.Seats { - for _, want := range seats { - if s.Seat != want { - continue - } - for _, h := range s.Holders { - if !seen[h.Node] { - seen[h.Node] = true - out = append(out, h.Node) - } + for _, seat := range seats { + holders, err := delivery.HoldersOf(keys, seat, "") + if err != nil { + return nil, err + } + for _, h := range holders { + if !seen[h.Node] { + seen[h.Node] = true + out = append(out, h.Node) } } } + if len(out) == 0 { + return nil, fmt.Errorf("the placement seat's state names no holder of %s", strings.Join(seats, " or ")) + } return out, nil } diff --git a/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go b/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go index 37052121..ddefa504 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go @@ -34,10 +34,10 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent}) { + if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent, "mesh-self-check.checked"}) { t.Fatalf("consumes %v", m.Consumes) } - if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping", "seat:mesh-controller.seats"}) { + if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping"}) { t.Fatalf("invokes %v", m.Invokes) } if len(m.Claims) != 0 { diff --git a/modules/mesh-watcher/cmd/mesh-watcher/takeover.go b/modules/mesh-watcher/cmd/mesh-watcher/takeover.go new file mode 100644 index 00000000..540d70c3 --- /dev/null +++ b/modules/mesh-watcher/cmd/mesh-watcher/takeover.go @@ -0,0 +1,59 @@ +package main + +import ( + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// takeover is which self-check run the watcher hears: the controller's `doctor-heartbeat` until the keeper's +// take-over key names the keeper, then mesh-self-check's `checked`, whose body is the same run (novox/hq ADR 0564 +// clause 9, issue 1952 W1). An absent key follows the controller. +type takeover struct { + mu sync.Mutex + t health.Takeover +} + +func newTakeover() *takeover { + return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}} +} + +// heard says whether key is the followed self-check event. +func (f *takeover) heard(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + seat, event := f.t.SelfCheckEvent() + return key == seat+"."+event +} + +func (f *takeover) set(value []byte, found bool) error { + t, err := health.DecodeTakeover(value, found) + if err != nil { + return err + } + f.mu.Lock() + f.t = t + f.mu.Unlock() + return nil +} + +// followTakeover watches the key for the life of the process; one it cannot read leaves the event it heard. +func followTakeover(f *takeover) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error { + if err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0); err != nil { + logf("[mesh-watcher] the take-over key is unreadable (%v); no change in what it hears", err) + } else { + logf("[mesh-watcher] the take-over key now follows %s", f.t.Follow) + } + return nil + }) + if err == nil { + return + } + logf("[mesh-watcher] not watching the take-over key yet (%v)", err) + time.Sleep(wait) + } +} diff --git a/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go b/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go index a335af11..bf77ce46 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "errors" "strings" "testing" @@ -214,16 +213,18 @@ func TestOnTheControlNodeItSaysItIsMisplaced(t *testing.T) { } } -func TestTheSeatsAnswerIsReadInEitherShape(t *testing.T) { - bare := `{"seats":[{"seat":"mesh-controller","holders":[{"node":"n1","module":"mesh-controller"}]}, - {"seat":"mesh-broker","holders":[{"node":"n1"},{"node":"n2"}]},{"seat":"git","holders":[{"node":"n3"}]}]}` - wrapped, _ := json.Marshal(map[string]any{"content": []map[string]string{{"type": "text", "text": bare}}}) - output, _ := json.Marshal(map[string]any{"ok": true, "output": bare}) - for _, raw := range []string{bare, string(wrapped), string(output)} { - got, err := holdersOf(json.RawMessage(raw), "mesh-controller", "mesh-broker") - if err != nil || strings.Join(got, ",") != "n1,n2" { - t.Errorf("%s: %v %v", raw, got, err) - } +func TestTheHoldersAreReadFromThePlacementSeatsState(t *testing.T) { + keys := map[string][]byte{ + "mesh-controller": []byte(`{"v":1,"seat":"mesh-controller","scope":"mesh","holders":[{"module":"mesh-controller","node":"n1"}]}`), + "mesh-broker": []byte(`{"v":1,"seat":"mesh-broker","scope":"mesh","holders":[{"module":"nats","node":"n1"},{"module":"nats","node":"n2"}]}`), + "git": []byte(`{"v":1,"seat":"git","scope":"mesh","holders":[{"module":"gitea","node":"n3"}]}`), + } + got, err := holderNodes(keys, "mesh-controller", "mesh-broker") + if err != nil || strings.Join(got, ",") != "n1,n2" { + t.Errorf("%v %v", got, err) + } + if _, err := holderNodes(map[string][]byte{}, "mesh-controller", "mesh-broker"); err == nil { + t.Error("no holder named, and no error") } } diff --git a/modules/mesh-watcher/go.mod b/modules/mesh-watcher/go.mod index 6c449fbb..37ea6996 100644 --- a/modules/mesh-watcher/go.mod +++ b/modules/mesh-watcher/go.mod @@ -1,5 +1,5 @@ module mesh-watcher -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/mesh-watcher/go.sum b/modules/mesh-watcher/go.sum index b474419e..09ad064e 100644 --- a/modules/mesh-watcher/go.sum +++ b/modules/mesh-watcher/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index e9543a5d..ac05e105 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -3,11 +3,15 @@ "version": "1", "slug": "watch", "consumes": [ - "mesh-controller.doctor-heartbeat" + "mesh-controller.doctor-heartbeat", + "mesh-self-check.checked" + ], + "reads": [ + "mesh-conditions.takeover", + "mesh-delivery-assigner.seat-holders" ], "invokes": [ - "mesh-watcher.watcher_ping", - "seat:mesh-controller.seats" + "mesh-watcher.watcher_ping" ], "own-secrets": { "telegram-token": "${dir:state}/telegram-token" @@ -39,7 +43,9 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", - "takes-config": ["telegram-chat-id"], + "takes-config": [ + "telegram-chat-id" + ], "merge": "json", "template": "files/settings.json" } diff --git a/modules/messenger/cmd/messenger/answers.go b/modules/messenger/cmd/messenger/answers.go index d92bf1af..6a3fbb5a 100644 --- a/modules/messenger/cmd/messenger/answers.go +++ b/modules/messenger/cmd/messenger/answers.go @@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int) // stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked. // Anything else is refused, and said there. The same answer is acted on once. func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) { - // The controller's root-free word is asked before the work is taken (the confirmation review of 2026-10-09): + // The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09): // up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer, // and judged with in it. root := h.rootWordFor(kind) diff --git a/modules/messenger/cmd/messenger/answers_test.go b/modules/messenger/cmd/messenger/answers_test.go index cdffa473..c1ff0d0f 100644 --- a/modules/messenger/cmd/messenger/answers_test.go +++ b/modules/messenger/cmd/messenger/answers_test.go @@ -600,6 +600,16 @@ func TestAnAskIsRefusedInWordsToItsAsker(t *testing.T) { if _, kept := store.asks["mesh-delivery.d-1"]; kept { t.Error("a refused ask was recorded") } + // A node's engine is an asker under node-engine- (novox/hq ADR 0563 §3, changing ADR 0259 §2): its ask is + // recorded under that name, with its own bound of open asks; a name that is no asker's is dropped. + submit(t, h, "node-engine-ace", heldAsk(c, "hand-over-1")) + submit(t, h, "node.ace", heldAsk(c, "hand-over-2")) + if rec, kept := store.asks["node-engine-ace.hand-over-1"]; !kept || rec.Asker != "node-engine-ace" { + t.Errorf("a node's engine's ask was not recorded under its name: %+v", rec) + } + if _, kept := store.asks["node.ace.hand-over-2"]; kept { + t.Error("an ask under a name that is no asker's was recorded") + } // No channel can carry anything: held with the reason (ADR 0397), never queued in silence. h2, _, _, c2, store2, w2 := newRouter(t) h2.Bench.Stand("telegram", Standing{Ready: false, Why: "no bot token", At: c2.now()}) diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index 08a53a95..b719e68f 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -242,8 +242,9 @@ func (h *Holder) carriers(level asks.Level) []string { return out } -// RootVerdict is the controller's word on one machine (its verb root-free, novox/hq ADR 0259 §8): free only -// when it measured, recently, that no agent can become root there without a person. +// RootVerdict is a machine's engine's word on that machine (the node seat node-engine's root-free, novox/hq ADR 0259 +// §8 as ADR 0563 §6 changes it): free only when it measured, recently, that no agent can become root there without a +// person. type RootVerdict struct { Machine string `json:"machine"` Free bool `json:"free"` @@ -251,51 +252,64 @@ type RootVerdict struct { Judged string `json:"judged,omitempty"` } -// RootFreeVerb is the controller's verb the router asks, as this module, which machines are root-free. -const RootFreeVerb = "seat:" + ControllerSeat + ".root-free" +// EngineSeat is the node seat each machine's engine holds, and RootFreeVerb its verb the router asks of a machine, as +// this module: `seat:node-engine.root-free@`, which that machine's engine alone answers (ADR 0563 §6). +const ( + EngineSeat = "node-engine" + RootFreeVerb = "seat:" + EngineSeat + ".root-free" +) -// RootFreeWithin is how long the router waits for the controller's word; RootFreeKept how long a word is -// used to render a message's buttons. An answer is always judged on a word asked then. +// RootFreeWithin is how long the router waits for the machines' words; RootFreeKept how long a word is used to render a +// message's buttons. An answer is always judged on a word asked then. const ( RootFreeWithin = 10 * time.Second RootFreeKept = 30 * time.Second ) -// askRootFree asks the controller about machines, failing on no answer in time. +// askRootFree asks each machine's own engine about that machine, at once, failing on no answer in time. A machine's +// word is the verdict its engine gives on it; a verdict an engine gives on another machine is not taken. func (h *Holder) askRootFree(machines []string) (map[string]RootVerdict, error) { if h.RootFree != nil { return h.RootFree(machines) } if h.Ask == nil { - return nil, errors.New("this router cannot ask the controller") + return nil, errors.New("this router cannot ask a machine's engine") } type answer struct { - raw json.RawMessage - err error + machine string + raw json.RawMessage + err error } - done := make(chan answer, 1) - go func() { - raw, err := h.Ask(RootFreeVerb, map[string]any{"machines": machines}) - done <- answer{raw, err} - }() - var a answer - select { - case a = <-done: - case <-time.After(RootFreeWithin): - return nil, fmt.Errorf("the controller did not answer within %s", RootFreeWithin) - } - if a.err != nil { - return nil, a.err - } - var body struct { - Machines []RootVerdict `json:"machines"` - } - if err := json.Unmarshal(a.raw, &body); err != nil { - return nil, fmt.Errorf("the controller's answer could not be read: %v", err) + done := make(chan answer, len(machines)) + for _, m := range machines { + go func(m string) { + raw, err := h.Ask(RootFreeVerb+"@"+m, map[string]any{}) + done <- answer{m, raw, err} + }(m) } out := map[string]RootVerdict{} - for _, v := range body.Machines { - out[v.Machine] = v + deadline := time.After(RootFreeWithin) + for range machines { + var a answer + select { + case a = <-done: + case <-deadline: + return nil, fmt.Errorf("a machine's engine did not answer within %s", RootFreeWithin) + } + if a.err != nil { + return nil, fmt.Errorf("%s's engine: %w", a.machine, a.err) + } + var body struct { + Machines []RootVerdict `json:"machines"` + } + if err := json.Unmarshal(a.raw, &body); err != nil { + return nil, fmt.Errorf("%s's engine's answer could not be read: %v", a.machine, err) + } + for _, v := range body.Machines { + if v.Machine == a.machine { + out[v.Machine] = v + } + } } return out, nil } @@ -329,7 +343,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { case cached: // Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree // asks outside it. - return "the controller's word on whether an agent can become root there is not recent" + return "the machines' engines' word on whether an agent can become root there is not recent" default: got, err := h.askRootFree(machines) if err != nil { @@ -351,7 +365,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { v, said := verdicts[m] switch { case !said: - return "the controller said nothing of whether an agent can become root on " + m + return "its engine said nothing of whether an agent can become root on " + m case !v.Free: why := v.Why if why == "" { @@ -364,7 +378,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { return "" } -// warmRootFree asks the controller's root-free word for every kind that proves its sender, outside the work, +// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work, // so what is rendered and offered inside it reads a recent word. // force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes. func (h *Holder) warmRootFree(force bool) { @@ -390,7 +404,7 @@ func (h *Holder) warmRootFree(force bool) { } } -// rootWordFor asks the controller's root-free word for a kind fresh, before the work is taken, and answers +// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers // what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none. // A kind that does not prove its sender is asked nothing: its answers never approve. func (h *Holder) rootWordFor(kind string) func(KindInfo) string { @@ -453,7 +467,7 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { if verb == ConditionVerb { // Before the work lock: Condition takes it, and decides as for the controller's conditions. if !asks.UsableID(strings.ReplaceAll(asker, "-", "_")) { - h.Logf("[messenger] a condition from %q was taken and dropped: not a module's name", asker) + h.Logf("[messenger] a condition from %q was taken and dropped: not an asker's name (a module's, or a node's engine's)", asker) return nil } h.takeCondition(asker, body) @@ -466,7 +480,7 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { h.init() h.mu.Unlock() if !asks.UsableID(strings.ReplaceAll(asker, "-", "_")) { - h.Logf("[messenger] an ask from %q was taken and dropped: not a module's name", asker) + h.Logf("[messenger] an ask from %q was taken and dropped: not an asker's name (a module's, or a node's engine's)", asker) return nil } if verb == "cancel" { diff --git a/modules/messenger/cmd/messenger/bus_test.go b/modules/messenger/cmd/messenger/bus_test.go index 9c67bda6..c42fb278 100644 --- a/modules/messenger/cmd/messenger/bus_test.go +++ b/modules/messenger/cmd/messenger/bus_test.go @@ -256,16 +256,8 @@ func (r *busRuntime) runtime(id json.RawMessage, method string, params json.RawM case "mesh/seat.publish": return map[string]any{}, r.nc.Publish(p.Subject, p.Body) case "mesh/ask": - if p.Key == RootFreeVerb { - var a struct { - Machines []string `json:"machines"` - } - _ = json.Unmarshal(p.Body, &a) - var out []RootVerdict - for _, m := range a.Machines { - out = append(out, RootVerdict{Machine: m, Free: true}) - } - return map[string]any{"machines": out}, nil + if m, ok := strings.CutPrefix(p.Key, RootFreeVerb+"@"); ok { + return map[string]any{"machines": []RootVerdict{{Machine: m, Free: true}}}, nil } return nil, fmt.Errorf("the test runtime answers nothing for %s", p.Key) } diff --git a/modules/messenger/cmd/messenger/condition.go b/modules/messenger/cmd/messenger/condition.go index 1378d596..7a218fad 100644 --- a/modules/messenger/cmd/messenger/condition.go +++ b/modules/messenger/cmd/messenger/condition.go @@ -122,9 +122,12 @@ func (c Condition) SilencedAt(now time.Time) bool { // eventOf is the local event name of an envelope's key: `mesh-controller.condition-raised` is // `condition-raised`. Anything not of the controller's seat is not a condition event. -func eventOf(key string) (string, bool) { +func eventOf(key string) (string, bool) { return eventOfSeat(key, ControllerSeat) } + +// eventOfSeat is eventOf for the seat the take-over key says is followed: the controller's, or the keeper's (ADR 0564 clause 9). +func eventOfSeat(key, seat string) (string, bool) { emitter, event, ok := strings.Cut(key, ".") - if !ok || emitter != ControllerSeat { + if !ok || emitter != seat { return "", false } switch event { diff --git a/modules/messenger/cmd/messenger/holder.go b/modules/messenger/cmd/messenger/holder.go index 120daa75..b3dff885 100644 --- a/modules/messenger/cmd/messenger/holder.go +++ b/modules/messenger/cmd/messenger/holder.go @@ -315,7 +315,8 @@ type Holder struct { Emit func(event string, body any) error // Ask calls a seat verb through the mesh, as this module: the answers the operator chooses. Ask Asker - // RootFree asks the controller's verb root-free about machines; nil asks it through Ask (askRootFree). + // RootFree asks each machine's engine's root-free (the node seat node-engine's, ADR 0563 §6); nil asks it through Ask + // (askRootFree). RootFree func(machines []string) (map[string]RootVerdict, error) // Zone is the operator's time zone, which every time in a message is said in; nil is this machine's. Zone func() *time.Location diff --git a/modules/messenger/cmd/messenger/main.go b/modules/messenger/cmd/messenger/main.go index e6c963f8..2cc92c95 100644 --- a/modules/messenger/cmd/messenger/main.go +++ b/modules/messenger/cmd/messenger/main.go @@ -18,6 +18,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" ) func errorf(format string, a ...any) error { return fmt.Errorf(format, a...) } @@ -415,8 +416,9 @@ func run(h *Holder, l *listening) { // The state first: what is open now is read from the controller before any event is taken, so // what the stream replays is history (novox/hq issue 271). Not reachable yet, it is asked for // again in the loop below, and old events meanwhile are kept as state, never said. + follow := newTakeover() h.Start() - syncFromController(h) + syncFromController(h, follow) go func() { tick := time.NewTicker(5 * time.Second) defer tick.Stop() @@ -429,10 +431,11 @@ func run(h *Holder, l *listening) { } if h.WantsSync() && now.Sub(lastAsk) >= time.Minute { lastAsk = now - syncFromController(h) + syncFromController(h, follow) } } }() + go followTakeover(follow, h) handle := func(e stdio.Envelope) error { if event, ok := strings.CutPrefix(e.Key, ViewModule+"."); ok { // A module in motion on a node, settled or stuck: a walk's start or end now that plans are gone (ADR 0308 §1, ADR 0539 §17). @@ -446,7 +449,9 @@ func run(h *Holder, l *listening) { } return nil } - event, ok := eventOf(e.Key) + // Only the seat the take-over key names is heard: the other's events are dropped, so the keeper's conditions + // are never held a second time beside the controller's (ADR 0564 clause 9). + event, ok := eventOfSeat(e.Key, follow.seat()) if !ok { return nil } @@ -476,6 +481,9 @@ func run(h *Holder, l *listening) { }() for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { err := stdio.Subscribe(ControllerSeat+".*", handle) + if err == nil { + err = stdio.Subscribe(health.KeeperModule+".*", handle) + } if err == nil { l.set("listening") logf("[messenger] listening for the controller's condition events") @@ -625,7 +633,7 @@ func followKinds(h *Holder) { } // syncFromController reads the controller's open conditions and gives them to the holder. -func syncFromController(h *Holder) { +func syncFromController(h *Holder, follow *takeover) { asked := h.Now() type answer struct { raw json.RawMessage @@ -633,7 +641,7 @@ func syncFromController(h *Holder) { } done := make(chan answer, 1) go func() { - raw, err := stdio.Ask("seat:"+ControllerSeat+".conditions", map[string]any{}) + raw, err := stdio.Ask("seat:"+follow.seat()+".conditions", map[string]any{}) done <- answer{raw, err} }() var a answer diff --git a/modules/messenger/cmd/messenger/manifest_test.go b/modules/messenger/cmd/messenger/manifest_test.go index 2281501b..6e61b0de 100644 --- a/modules/messenger/cmd/messenger/manifest_test.go +++ b/modules/messenger/cmd/messenger/manifest_test.go @@ -81,13 +81,14 @@ func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { // } if !reflect.DeepEqual(m.Consumes, []string{ ControllerSeat + "." + EventRaised, ControllerSeat + "." + EventChanged, ControllerSeat + "." + EventCleared, + "mesh-conditions." + EventRaised, "mesh-conditions." + EventChanged, "mesh-conditions." + EventCleared, ViewModule + "." + EventMoving, ViewModule + "." + EventSettled, ViewModule + "." + EventStuck, "intake.choice", "intake.link", "intake.failed", "intake.standing"}) { t.Fatalf("consumes %v", m.Consumes) } // It performs nothing: the verbs it calls read the open conditions, whether root is free, and which issues // track which conditions. if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:" + ControllerSeat + ".conditions", - "seat:" + ControllerSeat + ".root-free", TrackingVerb}) { + "seat:mesh-conditions.conditions", "seat:" + EngineSeat + ".root-free", TrackingVerb}) { t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes) } var state []string diff --git a/modules/messenger/cmd/messenger/review_test.go b/modules/messenger/cmd/messenger/review_test.go index 3409c3a6..9c042edc 100644 --- a/modules/messenger/cmd/messenger/review_test.go +++ b/modules/messenger/cmd/messenger/review_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "sync" "testing" @@ -66,26 +67,37 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { } } -// H3: the word is asked of the controller's verb, as this module, and read from its answer. -func TestTheRootWordIsAskedOfTheControllersVerb(t *testing.T) { +// H3, as ADR 0563 §6 changes it: the word on each machine is asked of that machine's own engine, the node seat +// node-engine's root-free there, as this module, and read from its answer; a machine's word is that machine's alone. +func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) { h, _, _, _, _, _ := newRouter(t) h.RootFree = nil var asked []string h.Ask = func(key string, body any) (json.RawMessage, error) { asked = append(asked, key) - raw, _ := json.Marshal(body) - if !strings.Contains(string(raw), `"machines":["anchor"]`) { - t.Errorf("asked about %s", raw) + machine := strings.TrimPrefix(key, "seat:node-engine.root-free@") + if machine == "elsewhere" { + machine = "anchor" // an engine that answers for another machine is not believed } - return json.RawMessage(`{"machines":[{"machine":"anchor","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil + return json.RawMessage(`{"machines":[{"machine":"` + machine + `","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil } info, _ := h.Bench.Kind("telegram") if why := h.rootFree(info, false); why != "" { t.Fatalf("free not read: %s", why) } - if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free" { + if len(asked) != 1 || asked[0] != "seat:node-engine.root-free@anchor" { t.Errorf("asked %v", asked) } + asked = nil + h.Node = "elsewhere" + if why := h.rootFree(info, false); !strings.Contains(why, "said nothing of whether an agent can become root on elsewhere") { + t.Errorf("one machine's word was taken for another's: %q", why) + } + sort.Strings(asked) + if len(asked) != 2 || asked[0] != "seat:node-engine.root-free@anchor" || asked[1] != "seat:node-engine.root-free@elsewhere" { + t.Errorf("the router's machine and the channel's were asked as %v", asked) + } + h.Node = "anchor" h.Ask = func(string, any) (json.RawMessage, error) { return json.RawMessage(`not json`), nil } if why := h.rootFree(info, false); !strings.Contains(why, "could not be read") { t.Errorf("an unreadable answer was a pass: %q", why) diff --git a/modules/messenger/cmd/messenger/sync.go b/modules/messenger/cmd/messenger/sync.go index efeaa6ee..34d1d696 100644 --- a/modules/messenger/cmd/messenger/sync.go +++ b/modules/messenger/cmd/messenger/sync.go @@ -180,3 +180,11 @@ func (h *Holder) WantsSync() bool { h.init() return h.wantSync || h.syncedAt.IsZero() || h.Now().Sub(h.syncedAt) >= ResyncEvery } + +// AskSync has the next pass of the loop read the open conditions again (the take-over key named another seat). +func (h *Holder) AskSync() { + h.mu.Lock() + h.init() + h.wantSync = true + h.mu.Unlock() +} diff --git a/modules/messenger/cmd/messenger/takeover.go b/modules/messenger/cmd/messenger/takeover.go new file mode 100644 index 00000000..e02ae421 --- /dev/null +++ b/modules/messenger/cmd/messenger/takeover.go @@ -0,0 +1,59 @@ +package main + +import ( + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// takeover is which seat's condition events the messenger hears: the controller's until the keeper's take-over key +// says otherwise (novox/hq ADR 0564 clause 9, issue 1952 R1). An absent key follows the controller. +type takeover struct { + mu sync.Mutex + t health.Takeover +} + +func newTakeover() *takeover { + return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}} +} + +func (f *takeover) seat() string { f.mu.Lock(); defer f.mu.Unlock(); return f.t.Seat() } + +// set takes the key as a watch handed it and says whether the followed seat changed. +func (f *takeover) set(value []byte, found bool) (changed bool, err error) { + t, err := health.DecodeTakeover(value, found) + if err != nil { + return false, err + } + f.mu.Lock() + defer f.mu.Unlock() + changed = f.t.Seat() != t.Seat() + f.t = t + return changed, nil +} + +// followTakeover watches the take-over key for the life of the process, and asks the holder to read the followed +// seat's open conditions again when it changes. A key it cannot read leaves the seat it followed. +func followTakeover(f *takeover, h *Holder) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error { + changed, err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0) + if err != nil { + logf("[messenger] the take-over key is unreadable (%v); still hearing %s", err, f.seat()) + return nil + } + if changed { + logf("[messenger] the take-over key now names %s", f.seat()) + h.AskSync() + } + return nil + }) + if err == nil { + return + } + logf("[messenger] not watching the take-over key yet (%v); hearing %s", err, f.seat()) + time.Sleep(wait) + } +} diff --git a/modules/messenger/go.mod b/modules/messenger/go.mod index e70013a7..9ea72bd8 100644 --- a/modules/messenger/go.mod +++ b/modules/messenger/go.mod @@ -3,7 +3,7 @@ module messenger go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/messenger/go.sum b/modules/messenger/go.sum index aa13456d..a2b8e018 100644 --- a/modules/messenger/go.sum +++ b/modules/messenger/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 h1:qHpN/fpTDqijfMVoepxUT6mPkfcydb4LiPKsIokv6iM= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/messenger/module.json b/modules/messenger/module.json index 4aceaebe..eec860f0 100644 --- a/modules/messenger/module.json +++ b/modules/messenger/module.json @@ -77,6 +77,9 @@ "mesh-controller.condition-raised", "mesh-controller.condition-changed", "mesh-controller.condition-cleared", + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed", + "mesh-conditions.condition-cleared", "mesh-delivery-view.moving", "mesh-delivery-view.settled", "mesh-delivery-view.stuck", @@ -90,9 +93,13 @@ ], "invokes": [ "seat:mesh-controller.conditions", - "seat:mesh-controller.root-free", + "seat:mesh-conditions.conditions", + "seat:node-engine.root-free", "seat:issue-tracker.tracking" ], + "reads": [ + "mesh-conditions.takeover" + ], "state": [ "open", "sent", diff --git a/modules/minio/cmd/object-store/manifest_test.go b/modules/minio/cmd/object-store/manifest_test.go index 34e5ebb9..58494c7f 100644 --- a/modules/minio/cmd/object-store/manifest_test.go +++ b/modules/minio/cmd/object-store/manifest_test.go @@ -20,7 +20,7 @@ func manifest(t *testing.T) map[string]any { return m } -// 1516/AC6: minio claims object-store (scope mesh) serving link, invokes seat:mesh-controller.seats, builds the Go +// 1516/AC6: minio claims object-store (scope mesh) serving link, reads the placement seat's seat-holders, builds the Go // bundle, and no tool named minio_presigned_url remains. func TestManifestClaimsTheSeat(t *testing.T) { m := manifest(t) @@ -38,9 +38,9 @@ func TestManifestClaimsTheSeat(t *testing.T) { if c["name"] != "object-store" || c["scope"] != "mesh" || len(serves) != 1 || serves[0] != "link" { t.Fatalf("claim: %v", c) } - inv := m["invokes"].([]any) - if len(inv) != 1 || inv[0] != "seat:mesh-controller.seats" { - t.Fatalf("invokes: %v", inv) + reads, _ := m["reads"].([]any) + if _, invokes := m["invokes"]; invokes || len(reads) != 1 || reads[0] != "mesh-delivery-assigner.seat-holders" { + t.Fatalf("reads %v and invokes %v: it reads the placement seat's holders and calls nothing", reads, m["invokes"]) } var goBundle map[string]any for _, a := range m["build"].(map[string]any)["artifacts"].([]any) { diff --git a/modules/minio/cmd/object-store/server.go b/modules/minio/cmd/object-store/server.go index 9b4b59e5..789873b8 100644 --- a/modules/minio/cmd/object-store/server.go +++ b/modules/minio/cmd/object-store/server.go @@ -13,6 +13,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/objectlink" ) @@ -59,11 +60,11 @@ func linkKeyID(bucket string) string { type server struct { configPath, linksDir, receives string now func() time.Time - // seats asks the controller for its seats; a test replaces it. - seats func() (json.RawMessage, error) + // seats reads the placement seat's state of who holds which seat, every key; a test replaces it. + seats func() (map[string][]byte, error) mu sync.Mutex - holders map[string]bool // "/." + keys map[string][]byte // seat-holders, as read readAt time.Time haveSet bool } @@ -77,8 +78,23 @@ func newServer(config, links, receives string) (*server, error) { return &server{configPath: config, linksDir: links, receives: receives, now: time.Now, seats: askSeats}, nil } -func askSeats() (json.RawMessage, error) { - return stdio.Ask("seat:mesh-controller.seats", map[string]any{}) +func askSeats() (map[string][]byte, error) { + state := stdio.State(delivery.SeatHoldersRead) + names, err := state.Keys() + if err != nil { + return nil, err + } + keys := map[string][]byte{} + for _, k := range names { + e, err := state.Get(k) + if err != nil { + return nil, err + } + if e != nil { + keys[k] = e.Value + } + } + return keys, nil } // link answers one ask. @@ -206,7 +222,7 @@ func (s *server) allowed(g grant, node, module, method string) error { if seat, isSeat := strings.CutPrefix(sh.To, "seat:"); isSeat { held, err := s.holds(seat, node, module) if err != nil { - return false, fmt.Errorf("a share names seat %s and the controller's seats cannot be read to tell whether this caller holds it: %w", seat, err) + return false, fmt.Errorf("a share names seat %s and the placement seat's holders cannot be read to tell whether this caller holds it: %w", seat, err) } return held, nil } @@ -249,58 +265,26 @@ func has(list []string, v string) bool { return false } -// holds says whether node.module holds the seat, from the controller's seats, read at most once a minute. A read that -// failed is not cached: the next ask reads again. +// holds says whether node.module holds the seat, from the placement seat's state of who holds which seat (novox/hq ADR +// 0571 clause 4), read at most once a minute. A read that failed is not cached: the next ask reads again. func (s *server) holds(seat, node, module string) (bool, error) { s.mu.Lock() defer s.mu.Unlock() if !s.haveSet || s.now().Sub(s.readAt) >= holdersFor { - raw, err := s.seats() + keys, err := s.seats() if err != nil { return false, err } - set, err := holdersIn(raw) - if err != nil { - return false, err - } - s.holders, s.readAt, s.haveSet = set, s.now(), true + s.keys, s.readAt, s.haveSet = keys, s.now(), true } - return s.holders[seat+"/"+node+"."+module], nil -} - -// holdersIn reads the controller's `seats` answer (answer.seats[].holders[] of {node, module}, and answer.outside[] of -// {Claim, Node, Module}) into "/.". The answer may arrive wrapped in {"answer": ...}. -func holdersIn(raw json.RawMessage) (map[string]bool, error) { - var doc struct { - Answer json.RawMessage `json:"answer"` - Seats []struct { - Seat string `json:"seat"` - Holders []struct { - Node string `json:"node"` - Module string `json:"module"` - } `json:"holders"` - } `json:"seats"` - Outside []struct { - Claim, Node, Module string - } `json:"outside"` + holders, err := delivery.HoldersOf(s.keys, seat, node) + if err != nil { + return false, err } - if err := json.Unmarshal(raw, &doc); err != nil { - return nil, fmt.Errorf("the controller's seats answer is not JSON: %w", err) - } - if len(doc.Answer) > 0 && string(doc.Answer) != "null" { - return holdersIn(doc.Answer) - } - if doc.Seats == nil && doc.Outside == nil { - return nil, errors.New("the controller's seats answer holds neither seats nor outside claims") - } - out := map[string]bool{} - for _, s := range doc.Seats { - for _, h := range s.Holders { - out[s.Seat+"/"+h.Node+"."+h.Module] = true + for _, h := range holders { + if h.Node == node && h.Module == module { + return true, nil } } - for _, o := range doc.Outside { - out[o.Claim+"/"+o.Node+"."+o.Module] = true - } - return out, nil + return false, nil } diff --git a/modules/minio/cmd/object-store/server_test.go b/modules/minio/cmd/object-store/server_test.go index 2f944a27..ba7bc956 100644 --- a/modules/minio/cmd/object-store/server_test.go +++ b/modules/minio/cmd/object-store/server_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "errors" "net/url" "os" @@ -25,12 +24,13 @@ const grantFile = `{ ] }` -// The seats answer is the controller's (mesh-controller.seats read on 2026-10-21): seats[].holders[] of {node, module}, -// and outside[] of {Claim, Node, Module}. -const seatsAnswer = `{"answer": {"seats": [ - {"seat": "node-files", "scope": "node", "holders": [{"node": "g14", "module": "mesh-files"}, {"node": "shanks", "module": "mesh-files"}]}, - {"seat": "node-backup", "scope": "node", "holders": [{"node": "g14", "module": "restic"}]} - ], "outside": [{"Claim": "printer", "Scope": "mesh", "Node": "shanks", "Module": "brother", "Site": "home", "Kind": ""}]}}` +// The placement seat's state of who holds which seat (seat-holders): a key per node seat and node, a key per mesh seat. +var seatHolderKeys = map[string][]byte{ + "node-files.g14": []byte(`{"v":1,"seat":"node-files","scope":"node","node":"g14","holders":[{"module":"mesh-files","node":"g14"}]}`), + "node-files.shanks": []byte(`{"v":1,"seat":"node-files","scope":"node","node":"shanks","holders":[{"module":"mesh-files","node":"shanks"}]}`), + "node-backup.g14": []byte(`{"v":1,"seat":"node-backup","scope":"node","node":"g14","holders":[{"module":"restic","node":"g14"}]}`), + "printer": []byte(`{"v":1,"seat":"printer","scope":"mesh","holders":[{"module":"brother","node":"shanks"}]}`), +} type rig struct { s *server @@ -64,12 +64,12 @@ func newRig(t *testing.T) *rig { t.Fatal(err) } s.now = func() time.Time { return r.now } - s.seats = func() (json.RawMessage, error) { + s.seats = func() (map[string][]byte, error) { r.seatsCalls++ if r.seatsErr != nil { return nil, r.seatsErr } - return json.RawMessage(seatsAnswer), nil + return seatHolderKeys, nil } r.s = s return r diff --git a/modules/minio/go.mod b/modules/minio/go.mod index c09ccfda..c3197341 100644 --- a/modules/minio/go.mod +++ b/modules/minio/go.mod @@ -2,4 +2,4 @@ module minio go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/minio/go.sum b/modules/minio/go.sum index 9728ce93..09ad064e 100644 --- a/modules/minio/go.sum +++ b/modules/minio/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/minio/module.json b/modules/minio/module.json index 01a966af..60b40a4b 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -12,6 +12,9 @@ "identity": { "max": 20, "in": "an S3 access key" + }, + "credential": { + "form": "pair" } } ], @@ -36,8 +39,8 @@ ] } ], - "invokes": [ - "seat:mesh-controller.seats" + "reads": [ + "mesh-delivery-assigner.seat-holders" ], "contributes": { "route": { diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index cbb12681..fc7d5212 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -12,6 +12,9 @@ "identity": { "max": 63, "in": "a MongoDB database name" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/mosquitto/module.json b/modules/mosquitto/module.json index 21058e0a..a7213d44 100644 --- a/modules/mosquitto/module.json +++ b/modules/mosquitto/module.json @@ -8,6 +8,9 @@ "scope": "mesh", "identity": { "in": "a Mosquitto client and topic prefix" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/mounts/cmd/mounts/bus.go b/modules/mounts/cmd/mounts/bus.go index ce3d0057..92505ce8 100644 --- a/modules/mounts/cmd/mounts/bus.go +++ b/modules/mounts/cmd/mounts/bus.go @@ -14,6 +14,7 @@ package main import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "time" @@ -122,22 +123,22 @@ func (m *Machine) leave(p string, v any) error { // Asker calls a tool on the mesh as this module. type Asker func(key string, body any) (json.RawMessage, error) -// DataVerb is the controller's verb that says where the mesh keeps data on a machine. -const DataVerb = "seat:mesh-controller.data" +// Readings reads disk-load's readings of one machine's data items (its state `readings`, ADR 0571 clause 3). +type Readings func(node string) ([]health.Reading, error) -// LeaveData asks the controller where the mesh keeps data on this machine and leaves the paths for the +// LeaveData reads disk-load for where the mesh keeps data on this machine and leaves the paths for the // process (finding 4d). A failed ask keeps the last paths, with why; with none, the process sets up // nothing new. -func (m *Machine) LeaveData(node string, ask Asker, now time.Time) []string { - if node == "" || ask == nil { +func (m *Machine) LeaveData(node string, read Readings, now time.Time) []string { + if node == "" || read == nil { return nil } previous, _ := readJSON[DataPaths](m, DataFile) out := DataPaths{Machine: node, Asked: now.Format(time.RFC3339)} - raw, err := ask(DataVerb, map[string]any{"machine": node}) + readings, err := read(node) var paths []string if err == nil { - paths, err = DataFrom(raw) + paths, err = PathsFrom(readings) } var said []string if err != nil { diff --git a/modules/mounts/cmd/mounts/data.go b/modules/mounts/cmd/mounts/data.go index 803a1a84..02fc6da1 100644 --- a/modules/mounts/cmd/mounts/data.go +++ b/modules/mounts/cmd/mounts/data.go @@ -6,9 +6,9 @@ package main // mesh keeps data in on this machine, from the controller's `data` verb over the bus. import ( - "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "io" "os" "path/filepath" @@ -63,44 +63,16 @@ func sortWants(w []Want) { } } -// DataFrom reads the paths out of mesh-controller.data's answer: `answer.data`, `data`, or the same as -// text in `output`. An answer with no data list is an error, never an empty list: an empty list would -// let every mount point through. -func DataFrom(raw json.RawMessage) ([]string, error) { - type item struct { - Path string `json:"path"` - } - var whole struct { - Answer *struct { - Data *[]item `json:"data"` - } `json:"answer"` - Data *[]item `json:"data"` - Output string `json:"output"` - } - if err := json.Unmarshal(raw, &whole); err != nil { - return nil, fmt.Errorf("the controller's data answer is not JSON: %v", err) - } - var items *[]item - switch { - case whole.Answer != nil && whole.Answer.Data != nil: - items = whole.Answer.Data - case whole.Data != nil: - items = whole.Data - case whole.Output != "": - var inner struct { - Data *[]item `json:"data"` - } - if err := json.Unmarshal([]byte(whole.Output), &inner); err == nil && inner.Data != nil { - items = inner.Data - } - } - if items == nil { - return nil, errors.New("the controller's data answer names no data list") +// PathsFrom is the paths disk-load's readings name: where the mesh keeps data here. No reading is an error, never an +// empty list: an empty list would let every mount point through. +func PathsFrom(readings []health.Reading) ([]string, error) { + if len(readings) == 0 { + return nil, errors.New("disk-load keeps no reading of this machine's data yet") } paths := []string{} - for _, it := range *items { - if strings.HasPrefix(it.Path, "/") { - paths = append(paths, filepath.Clean(it.Path)) + for _, r := range readings { + if strings.HasPrefix(r.Path, "/") { + paths = append(paths, filepath.Clean(r.Path)) } } return paths, nil diff --git a/modules/mounts/cmd/mounts/main.go b/modules/mounts/cmd/mounts/main.go index ea5b32c0..7a69b8d0 100644 --- a/modules/mounts/cmd/mounts/main.go +++ b/modules/mounts/cmd/mounts/main.go @@ -13,8 +13,10 @@ import ( "context" "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "os/signal" + "strings" "syscall" "time" @@ -62,13 +64,36 @@ func shareOnTheBus(m *Machine) { _, err := stdio.State(SharesState).Put(key, value) return err } - ask := func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) } + read := func(node string) ([]health.Reading, error) { + st := stdio.State(health.ReadingsRead) + keys, err := st.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := st.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil + } last, lastSaid := "", "" for n := 0; ; n++ { said := m.Share(node, get, put, &last, time.Now()) // Where the mesh keeps data here changes only with an assignment: asked at start and every five minutes. if n%10 == 0 { - said = append(said, m.LeaveData(node, ask, time.Now())...) + said = append(said, m.LeaveData(node, read, time.Now())...) } if s := fmt.Sprint(said); len(said) > 0 && s != lastSaid { fmt.Fprintln(os.Stderr, "mounts:", s) diff --git a/modules/mounts/cmd/mounts/manifest_test.go b/modules/mounts/cmd/mounts/manifest_test.go index 257ec6ce..43c6f67e 100644 --- a/modules/mounts/cmd/mounts/manifest_test.go +++ b/modules/mounts/cmd/mounts/manifest_test.go @@ -77,12 +77,10 @@ func TestTheManifestNamesWhatTheBundleServesAndWhatItNeeds(t *testing.T) { if strings.Join(m.Requires, ",") != "nfs-share" { t.Errorf("requires %v", m.Requires) } - if strings.Join(m.Reads, ",") != ExportsState { - t.Errorf("the module reads nfs-server's exports from the bus: reads %v", m.Reads) - } - // It calls one verb: the controller's, for where the mesh keeps data on this machine (review finding 4d). - if strings.Join(m.Invokes, ",") != DataVerb { - t.Errorf("invokes %v, want %s", m.Invokes, DataVerb) + // It reads nfs-server's exports, and disk-load's readings for where the mesh keeps data on this machine (review + // finding 4d, ADR 0571 clause 3); it calls no verb. + if strings.Join(m.Reads, ",") != "disk-load.readings,"+ExportsState || len(m.Invokes) != 0 { + t.Errorf("reads %v, invokes %v", m.Reads, m.Invokes) } if !strings.Contains(raw, `node=${machine:name}`) { t.Error("the config gives the process this machine's name, so it never mounts a share onto the machine that serves it") diff --git a/modules/mounts/cmd/mounts/review_test.go b/modules/mounts/cmd/mounts/review_test.go index 9e1c4c68..34778bf1 100644 --- a/modules/mounts/cmd/mounts/review_test.go +++ b/modules/mounts/cmd/mounts/review_test.go @@ -5,6 +5,8 @@ package main import ( "context" "encoding/json" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "strings" @@ -198,18 +200,13 @@ func TestWithoutTheMeshsDataNothingNewIsSetUp(t *testing.T) { func TestTheBundleLeavesTheMeshsDataForTheProcess(t *testing.T) { f := newSystem(t) f.write(ConfigFile, libraryOnly) - answer := json.RawMessage(`{"answer":{"data":[{"machine":"workstation","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`) - paths, err := DataFrom(answer) + rs := []health.Reading{{V: 1, Node: "workstation", PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}} + paths, err := PathsFrom(rs) if err != nil || len(paths) != 1 || paths[0] != "/mnt/plex/data" { t.Fatalf("paths %q %v", paths, err) } - // The controller's verbs also answer as text in output. - paths, err = DataFrom(json.RawMessage(`{"ok":true,"output":"{\"data\":[{\"path\":\"/srv/x\"}]}"}`)) - if err != nil || len(paths) != 1 || paths[0] != "/srv/x" { - t.Fatalf("paths from output %q %v", paths, err) - } - if _, err := DataFrom(json.RawMessage(`{"ok":false}`)); err == nil { - t.Error("an answer with no data list is an error, not an empty list") + if _, err := PathsFrom(nil); err == nil { + t.Error("no reading is an error, not an empty list") } } @@ -287,23 +284,17 @@ func callVerb(t *testing.T, m *Machine, name string, args map[string]any) (any, func TestTheBundleKeepsTheLastDataWhenAnAskFails(t *testing.T) { f := newSystem(t) m := f.machine() - var asked []string - ok := func(key string, body any) (json.RawMessage, error) { - raw, _ := json.Marshal(body) - asked = append(asked, key+" "+string(raw)) - return json.RawMessage(`{"answer":{"data":[{"path":"/mnt/plex/data"}]}}`), nil + ok := func(node string) ([]health.Reading, error) { + return []health.Reading{{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}}, nil } if said := m.LeaveData("workstation", ok, time.Now()); len(said) != 0 { t.Fatalf("said %q", said) } - if len(asked) != 1 || asked[0] != `seat:mesh-controller.data {"machine":"workstation"}` { - t.Errorf("asked %q", asked) - } - failing := func(string, any) (json.RawMessage, error) { return nil, os.ErrDeadlineExceeded } + failing := func(string) ([]health.Reading, error) { return nil, os.ErrDeadlineExceeded } said := m.LeaveData("workstation", failing, time.Now()) d, err := readJSON[DataPaths](m, DataFile) if err != nil || len(d.Paths) != 1 || d.Paths[0] != "/mnt/plex/data" || d.Error == "" || len(said) == 0 { - t.Errorf("a failed ask keeps the last paths and says why: %+v %v %q", d, err, said) + t.Errorf("a failed read keeps the last paths and says why: %+v %v %q", d, err, said) } } diff --git a/modules/mounts/go.mod b/modules/mounts/go.mod index 76732a34..896fcdc5 100644 --- a/modules/mounts/go.mod +++ b/modules/mounts/go.mod @@ -1,5 +1,5 @@ module mounts -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/mounts/go.sum b/modules/mounts/go.sum index e655124d..f8bfbd2b 100644 --- a/modules/mounts/go.sum +++ b/modules/mounts/go.sum @@ -1,8 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mounts/module.json b/modules/mounts/module.json index 488ebb97..dc1e3785 100644 --- a/modules/mounts/module.json +++ b/modules/mounts/module.json @@ -5,11 +5,9 @@ "nfs-share" ], "reads": [ + "disk-load.readings", "nfs-server.exports" ], - "invokes": [ - "seat:mesh-controller.data" - ], "capabilities": [ "package-manager", "service-manager" @@ -38,20 +36,20 @@ "per-machine": true } ], - "config": { - "shares": { - "type": "string", - "description": "The shares of another machine this machine mounts: space-separated name=mountpoint, each optionally :ro, or none (novox/hq ADR 0263 rule 5).", - "trusted": true, - "default": "none" - }, - "sources": { - "type": "string", - "description": "The occasional sources this machine mounts: space-separated name=smb://[@]/@ or name=disk:@, each optionally :ro, or none (novox/hq ADR 0263 rule 6); an SMB source's username is in its entry, its password the secret smb-password- (novox/hq ADR 0283).", - "trusted": true, - "default": "none" - } + "config": { + "shares": { + "type": "string", + "description": "The shares of another machine this machine mounts: space-separated name=mountpoint, each optionally :ro, or none (novox/hq ADR 0263 rule 5).", + "trusted": true, + "default": "none" }, + "sources": { + "type": "string", + "description": "The occasional sources this machine mounts: space-separated name=smb://[@]/@ or name=disk:@, each optionally :ro, or none (novox/hq ADR 0263 rule 6); an SMB source's username is in its entry, its password the secret smb-password- (novox/hq ADR 0283).", + "trusted": true, + "default": "none" + } + }, "own-secrets": { "smb-password-*": { "path": "${dir:state}/smb-password-*.secret", diff --git a/modules/mssql/module.json b/modules/mssql/module.json index ed4e4488..35cdbe6e 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -12,6 +12,9 @@ "identity": { "max": 128, "in": "a SQL Server login and database name" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/nats/cmd/mesh-nats-snapshot/certificate.go b/modules/nats/cmd/mesh-nats-snapshot/certificate.go new file mode 100644 index 00000000..7d03bc9e --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/certificate.go @@ -0,0 +1,174 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +// The bus's own certificate, made by the mesh rather than borrowed from an image. Moved here from mesh-controller's +// `broker certificate` with the same output (design 47's L3 s11): the command belongs to the bus module, and the +// controller's binary is retired after its lease is handed away (ADR 0547 §11). Nothing changes at the lease write: the +// certificate in /var/lib/mesh-broker-tls stays, and every pin stays valid. +// +// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq +// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's +// image, which worked while the broker was one that happened to carry it and stopped the day the +// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be +// raised. Substituting another image the bundle names does not help — none of them carry it +// either. +// +// So the program that needs a certificate makes one. It is the mesh's own binary, already on the +// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the +// image it writes into but a mounted directory. +// +// **Self-signed, and that is the design** — a host pins this server's exact certificate and +// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at +// this moment in a bootstrap there is no mesh to ask one of. +// +// Idempotent, because the step is applied again on every reconcile and a second certificate would +// be one the hosts that pinned the first no longer believe. + +// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the +// loopback address the machine's own foundation dials. +var busCertificateNames = []string{"mesh-broker"} + +const busCertificateLife = 10 * 365 * 24 * time.Hour + +// busCertificate makes the bus's certificate in a directory, or says whether one is there. +// +// mesh-nats-snapshot certificate --into /tls make it if it is not there +// mesh-nats-snapshot certificate --check --into /tls exit non-zero unless a usable pair is +func busCertificate(args []string) error { + into, check := "", false + for i := 0; i < len(args); i++ { + switch args[i] { + case "--check": + check = true + case "--into": + if i+1 >= len(args) { + return errors.New("--into needs a directory") + } + into = args[i+1] + i++ + default: + return fmt.Errorf("certificate [--check] --into : %q", args[i]) + } + } + if into == "" { + return errors.New("certificate [--check] --into ") + } + crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key") + + if usable, err := busCertificateUsable(crt, key); err != nil { + return err + } else if usable { + fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt) + return nil + } + if check { + // Said as a failure, because that is what the caller asked: a bootstrap's verify runs + // this and a false answer is what makes the step run. + return fmt.Errorf("no usable certificate and key at %s", into) + } + return writeBusCertificate(crt, key) +} + +// busCertificateUsable says whether a certificate and its key are both there and parse. +// +// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted +// between the two files leaves behind, and a step that treated it as done would hand the server a +// certificate with no key and report success. +func busCertificateUsable(crt, key string) (bool, error) { + certPEM, err := os.ReadFile(crt) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + keyPEM, err := os.ReadFile(key) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if _, err := tlsPairParses(certPEM, keyPEM); err != nil { + return false, nil + } + return true, nil +} + +func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(certPEM) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("not a certificate") + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, errors.New("not a key") + } + if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil { + if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil { + return nil, err + } + } + return certificate, nil +} + +func writeBusCertificate(crt, key string) error { + private, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return err + } + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: busCertificateNames[0]}, + DNSNames: busCertificateNames, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(busCertificateLife), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private) + if err != nil { + return err + } + pkcs8, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + return err + } + + // **The key first, and only then the certificate**, so the pair a reader finds is never a + // certificate whose key has not been written yet — the one order in which an interruption + // leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable). + if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil { + return err + } + if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil { + return err + } + fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n", + busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key) + return nil +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go b/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go new file mode 100644 index 00000000..e03df592 --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "os" + "path/filepath" + "strings" + "testing" +) + +// L3 s11 (design 47): the bus's certificate command, moved from mesh-controller with the same output, made once, and +// its check false without a usable pair. Its cases, as the controller's tests had them, are rows of one test. +func TestTheBusCertificate(t *testing.T) { + for _, row := range []struct { + name string + run func(*testing.T) + }{ + {"it loads as a server loads it", loadsAsAServersWould}, + {"it is made once", isMadeOnce}, + {"the check is false until there is a pair", checkIsFalseUntilAPair}, + {"a certificate without its key is not usable", withoutItsKeyIsNotUsable}, + {"where to write is required", whereToWriteIsRequired}, + } { + t.Run(row.name, row.run) + } +} + +// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for +// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is +// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once. + +func loadsAsAServersWould(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatalf("the bus could not be given a certificate: %v", err) + } + + // The check a cheaper test would not make. The key was present and valid and the server could + // not start, once, because nothing loaded the pair the way a server loads it + // (novox/hq 04-ISSUES/014). + pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")) + if err != nil { + t.Fatalf("a TLS server cannot load what was written: %v", err) + } + leaf := pair.Leaf + if leaf == nil { + if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil { + t.Fatal(err) + } + } + if err := leaf.VerifyHostname("mesh-broker"); err != nil { + t.Errorf("the certificate is not for the name the bus is reached by: %v", err) + } + if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" { + t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses) + } + + // The key is not readable by anything else on the machine; the certificate is public and is. + key, err := os.Stat(filepath.Join(into, "tls.key")) + if err != nil { + t.Fatal(err) + } + if key.Mode().Perm() != 0o600 { + t.Errorf("the key is %v", key.Mode().Perm()) + } + crt, err := os.Stat(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if crt.Mode().Perm() != 0o644 { + t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm()) + } +} + +// **Made once.** The step is applied again on every reconcile, and a second certificate is one the +// hosts that pinned the first no longer believe (novox/hq ADR 0004). +func isMadeOnce(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + again, err := os.ReadFile(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if string(first) != string(again) { + t.Fatal("running it twice replaced the certificate every host had pinned") + } +} + +// The verify half: false before, true after, which is what makes the bootstrap run the step at all. +func checkIsFalseUntilAPair(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--check", "--into", into}); err == nil { + t.Fatal("an empty directory reported a usable certificate") + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--check", "--into", into}); err != nil { + t.Fatalf("the certificate it just made does not satisfy its own check: %v", err) + } +} + +// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that +// called it done would hand the server a certificate with no key and report success. +func withoutItsKeyIsNotUsable(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(into, "tls.key")); err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--check", "--into", into}); err == nil { + t.Fatal("a certificate with no key passed the check") + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil { + t.Fatalf("it did not replace the unusable pair: %v", err) + } +} + +func whereToWriteIsRequired(t *testing.T) { + if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") { + t.Fatalf("it did not ask where to write: %v", err) + } +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/connect.go b/modules/nats/cmd/mesh-nats-snapshot/connect.go index 97aad5b7..246a34fb 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/connect.go +++ b/modules/nats/cmd/mesh-nats-snapshot/connect.go @@ -1,105 +1,16 @@ package main import ( - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "strings" - "time" + "nats-tools/internal/busconn" "github.com/nats-io/nats.go" ) -// Credential is what the mesh seals to a module so it can reach the bus (mesh-controller -// cmd/mesh-controller issueWith): where, the server's certificate by fingerprint, who, and the -// password. The address is not used here — this runs beside the server, and the server is reached on -// loopback — but the fingerprint is: loopback or not, the certificate is the mesh's own. -type Credential struct { - URL string `json:"url"` - Fingerprint string `json:"fingerprint,omitempty"` - User string `json:"user"` - Password string `json:"password"` -} +// Credential is what the mesh seals to a module so it can reach the bus; see busconn. +type Credential = busconn.Credential -func readCredential(path string) (Credential, error) { - var raw []byte - var err error - if path == "-" { - raw, err = io.ReadAll(io.LimitReader(os.Stdin, 1<<20)) - } else { - raw, err = os.ReadFile(path) - } - if err != nil { - return Credential{}, fmt.Errorf("cannot read the credential: %w", err) - } - var c Credential - if err := json.Unmarshal(raw, &c); err != nil { - return Credential{}, errors.New("the credential is not the JSON the mesh seals to a module (url, fingerprint, user, password)") - } - if c.User == "" || c.Password == "" { - return Credential{}, errors.New("the credential names no user or password, so the bus would refuse it") - } - return c, nil -} +func readCredential(path string) (Credential, error) { return busconn.Read(path) } -// connect dials the bus as the credential's user. A `tls://` server is held to the credential's -// fingerprint and nothing else — the mesh's bus presents a certificate of its own, in no trust store, -// so a name check could only fail or be skipped. A plain `nats://` server is accepted only on -// loopback, and is what a restore's own private server is. func connect(server string, c Credential) (*nats.Conn, error) { - opts := []nats.Option{ - nats.UserInfo(c.User, c.Password), - nats.Name("mesh-nats-snapshot"), - // Answers come to the user's own inbox and no other: the bus grants every user its own - // `_INBOX..>` and nothing wider (novox/hq design 25 §4). - nats.CustomInboxPrefix("_INBOX." + c.User), - nats.Timeout(10 * time.Second), - // One attempt: a snapshot that cannot reach the bus fails the night, loudly, rather than - // waiting for it. - nats.NoReconnect(), - } - switch { - case strings.HasPrefix(server, "tls://"): - if c.Fingerprint == "" { - return nil, errors.New("the credential carries no fingerprint, so the bus's certificate cannot be checked") - } - opts = append(opts, nats.Secure(pinnedTo(c.Fingerprint))) - case strings.HasPrefix(server, "nats://127.0.0.1:"), strings.HasPrefix(server, "nats://localhost:"): - if c.Fingerprint != "" { - // The mesh's bus requires TLS; a plain connection to it would be refused anyway. - opts = append(opts, nats.Secure(pinnedTo(c.Fingerprint))) - } - default: - return nil, fmt.Errorf("%s: the bus is reached over tls:// (or plain nats:// on loopback, for a restore's own server)", server) - } - nc, err := nats.Connect(server, opts...) - if err != nil { - return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", server, c.User, err) - } - return nc, nil -} - -// pinnedTo accepts exactly the certificate with this SHA-256 — the same pin every machine holds -// (mesh-controller internal/broker PinnedToFingerprint). -func pinnedTo(want string) *tls.Config { - return &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter - MinVersion: tls.VersionTLS12, - VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { - if len(rawCerts) == 0 { - return errors.New("the bus presented no certificate") - } - sum := sha256.Sum256(rawCerts[0]) - if got := "sha256:" + hex.EncodeToString(sum[:]); got != want { - return fmt.Errorf("the bus presented a certificate this mesh does not know (%s)", got) - } - return nil - }, - } + return busconn.Connect(server, c, "mesh-nats-snapshot") } diff --git a/modules/nats/cmd/mesh-nats-snapshot/main.go b/modules/nats/cmd/mesh-nats-snapshot/main.go index f71e49d8..a3232c77 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/main.go +++ b/modules/nats/cmd/mesh-nats-snapshot/main.go @@ -17,6 +17,7 @@ // mesh-nats-snapshot verify [--from FILE] // mesh-nats-snapshot restore --into DIR [--account MESH] [--from FILE] // mesh-nats-snapshot restore --server URL --credential FILE [--from FILE] +// mesh-nats-snapshot certificate [--check] --into DIR // // **Read-only on the live bus.** A snapshot neither pauses nor reconfigures a stream; the server // keeps accepting writes while it streams the copy out. The credential it uses is the bus's own @@ -47,6 +48,8 @@ func main() { err = runVerify(os.Args[2:]) case "restore": err = runRestore(os.Args[2:]) + case "certificate": + err = busCertificate(os.Args[2:]) case "-h", "--help", "help": usage() return @@ -67,6 +70,7 @@ func usage() { verify [--from bus.tar] every archive against the manifest restore --into DIR [--account MESH] [--from bus.tar] into a new store directory, by a server of its own restore --server URL --credential FILE [--from bus.tar] into a running server that holds none of them + certificate [--check] --into DIR the bus's self-signed certificate, made once The credential is the JSON the mesh seals to the bus's own module: url, fingerprint, user, password. `) diff --git a/modules/nats/cmd/nats-tools/bus.go b/modules/nats/cmd/nats-tools/bus.go new file mode 100644 index 00000000..ea37bc88 --- /dev/null +++ b/modules/nats/cmd/nats-tools/bus.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "errors" + "fmt" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go/jetstream" + "nats-tools/internal/busconn" +) + +// LiveBus is the bus as the module's own credential reaches it. Nothing connects until a tool that changes the +// bus is called, so the tools that read (through the monitoring endpoint) answer as they did with no credential. +// NATS_TOOLS_CREDENTIAL names the credential (a word of the bundle's own: the runtime strips MESH_BROKER_FILE) (the process nats-tools's own secret `broker-nats-tools`); MESH_NATS_SERVER where the bus is as +// seen from this machine (default: its own port on loopback). +func LiveBus() Bus { + return func(ctx context.Context) (jetstream.JetStream, func(), error) { + file := env("NATS_TOOLS_CREDENTIAL", "") + if file == "" { + return nil, nil, errors.New("NATS_TOOLS_CREDENTIAL is not set, so this tool has no credential to reach the bus with (the module's manifest sets it)") + } + c, err := busconn.Read(file) + if err != nil { + return nil, nil, fmt.Errorf("the credential at %s (NATS_TOOLS_CREDENTIAL) cannot be read: %w", file, err) + } + nc, err := busconn.Connect(env("MESH_NATS_SERVER", "tls://127.0.0.1:4222"), c, "nats-tools") + if err != nil { + return nil, nil, err + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, nil, fmt.Errorf("JetStream on the bus: %w", err) + } + return js, nc.Close, nil + } +} + +// SaidByTheRuntime is the audit line: an event of this module, published through the runtime that launched it. +func SaidByTheRuntime(event string, body any) error { return stdio.Emit(event, body) } diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 4de769dd..933abbea 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -1,6 +1,7 @@ // nats-tools: the mesh bus's own tools (novox/hq design 25), a Go bundle the node's runtime launches on the -// machine that runs the bus. Everything here reads — the server's monitoring API and the user list the -// controller composed — and nothing changes the bus. stdout is the MCP channel; this says nothing else. +// machine that runs the bus. Almost everything here reads — the server's monitoring API and the user list the +// controller composed. The two that change the bus (removal.go) are the only ones that connect to it, on the +// module's own credential, and say what they did. stdout is the MCP channel; this says nothing else. package main import ( @@ -86,7 +87,7 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written return Subscriptions(subsz, connz, arg(a, "subject"), limitOf(a, 100)) }}, {Name: "nats_streams", - Description: "JetStream: every stream with its subjects, retention, message count and size — or, given one stream, every consumer on it with what it has pending, unacknowledged and redelivered.", + Description: "JetStream: every stream with its subjects, retention, message count, size and size cap (max_bytes, when it has one) — or, given one stream, every consumer on it with what it has pending, unacknowledged and redelivered.", Input: map[string]any{"stream": str("one stream, e.g. EVENTS, CONTROL, ASSIGNMENTS, SEAT_NODE_BUILD_AGENT or KV_")}, Run: func(a map[string]any) (any, error) { doc, err := get(jsz) @@ -151,7 +152,7 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written } func main() { - if err := stdio.Serve("", tools(LiveMonitor(), ReadUsers, UsersWritten)); err != nil { + if err := stdio.Serve("", append(tools(LiveMonitor(), ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime)...)); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index b9d699d0..2007b1ef 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "os" + "slices" "strings" "testing" ) @@ -24,11 +25,14 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { } `json:"backup"` } `json:"own"` } `json:"data"` + Tools []string `json:"tools"` + Emits []string `json:"emits"` Resources []map[string]any `json:"resources"` Build struct { On any `json:"on"` Artifacts []struct { Name, Kind, From, Language, System, Binary string + Env map[string]string } `json:"artifacts"` } `json:"build"` } @@ -38,7 +42,10 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { if m.Build.On != nil { t.Error("build.on is declared: nothing is built on a base image any more") } - artifacts := map[string]struct{ Name, Kind, From, Language, System, Binary string }{} + artifacts := map[string]struct { + Name, Kind, From, Language, System, Binary string + Env map[string]string + }{} for _, a := range m.Build.Artifacts { artifacts[a.Name] = a if a.Kind == "image" { @@ -103,6 +110,23 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("%s is not in the container's restart-on: a new program would leave the old one running", p.resource) } } + // The tools that change the bus (issue 1952 N1-N3): listed, their audit events declared, and the credential + // they dial with the module's own secret `broker`. + for _, want := range []string{"nats_consumer_remove", "nats_bucket_remove"} { + if !slices.Contains(m.Tools, want) { + t.Errorf("tools do not list %s", want) + } + } + for _, want := range []string{"consumer-removed", "bucket-keys-removed"} { + if !slices.Contains(m.Emits, want) { + t.Errorf("emits do not list %s: the removal tools say it as the audit line", want) + } + } + for _, a := range m.Build.Artifacts { + if a.Name == "tools" && a.Env["NATS_TOOLS_CREDENTIAL"] != "${dir:tools-secret}/broker-nats-tools" { + t.Errorf("the tools' credential is %q: the process nats-tools' own secret broker-nats-tools, under a word of the bundle's own (the runtime strips MESH_BROKER_FILE)", a.Env["NATS_TOOLS_CREDENTIAL"]) + } + } args, _ := server["args"].([]any) if len(args) != 2 || args[0] != "sh" || args[1] != "/opt/mesh-entrypoint/entrypoint.sh" { t.Errorf("args are %v: the entrypoint run through sh, so no execute bit is relied on", args) diff --git a/modules/nats/cmd/nats-tools/removal.go b/modules/nats/cmd/nats-tools/removal.go new file mode 100644 index 00000000..3aff66e5 --- /dev/null +++ b/modules/nats/cmd/nats-tools/removal.go @@ -0,0 +1,246 @@ +package main + +// The two tools of this module that change the bus (novox/hq ADR 0564 clause 5, issue 1952 notes 6, 7 and 9): +// remove one named consumer, and remove one node's keys under a prefix of a key-value bucket. Everything else +// here reads. +// +// **Narrow by what the tool needs, and narrower by what the bus grants.** The tools reach the bus on the +// credential the module's manifest names (NATS_TOOLS_CREDENTIAL); the bus refuses whatever that credential is not +// granted, so a wrong stream, consumer or bucket is refused by the server and said here, not trusted to the +// arguments. Each tool refuses without `why`, reads what it would remove first (and says it, so a person can ask +// with dry_run and then for real), removes, reads that it is gone, and says what it did as an event of the module +// (the audit trail: who asked is in the call the controller audits, why and what are in the event). + +import ( + "context" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// Bus opens JetStream on the bus as the module's credential for one call; close ends the connection. +type Bus func(ctx context.Context) (js jetstream.JetStream, close func(), err error) + +// Said keeps the audit line: an event of this module. +type Said func(event string, body any) error + +const ( + consumerRemoved = "consumer-removed" + bucketKeysRemoved = "bucket-keys-removed" + // maxKeysShown bounds the keys an answer lists; the count is always whole. + maxKeysShown = 200 +) + +var ( + // A stream, consumer or bucket is one subject token: no dot, no wildcard, no space. + name = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + // A key prefix is subject tokens and their dots; no wildcard, so it names keys and not a pattern. + keyPrefix = regexp.MustCompile(`^[A-Za-z0-9_./=-]+$`) +) + +func why(a map[string]any) (string, error) { + w := strings.TrimSpace(arg(a, "why")) + if w == "" { + return "", errors.New("why is required: say why this is removed, for the audit trail (novox/hq ADR 0564 clause 5); nothing was removed") + } + return w, nil +} + +// answerWithin is how long the bus is given to answer one request. **The bus does not refuse a request its +// credential is not granted: it drops it** (a permissions violation is told to the connection's error handler, +// never answered), so an unanswered request is how a refusal shows here. +var answerWithin = 10 * time.Second + +// unanswered says what a request with no answer most likely is. +func unanswered(err error, what string) error { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) { + return fmt.Errorf("%s: the bus did not answer within %s; it drops, without an answer, what the module's credential is not granted, "+ + "so this is most likely outside its grant (or the bus is not reachable): %w", what, answerWithin, err) + } + return fmt.Errorf("%s: %w", what, err) +} + +func step(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, answerWithin) +} + +// ConsumerRemoved is what nats_consumer_remove answers and says. +type ConsumerRemoved struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Why string `json:"why"` + Was map[string]any `json:"was"` + Removed bool `json:"removed"` + At time.Time `json:"at"` +} + +// RemoveConsumer removes one named consumer of one stream, after reading it, and says so. +func RemoveConsumer(ctx context.Context, bus Bus, said Said, a map[string]any) (any, error) { + w, err := why(a) + if err != nil { + return nil, err + } + stream, consumer := arg(a, "stream"), arg(a, "consumer") + if !name.MatchString(stream) || !name.MatchString(consumer) { + return nil, errors.New("stream and consumer are required, each one name (letters, digits, - and _): nothing was removed") + } + js, closeBus, err := bus(ctx) + if err != nil { + return nil, err + } + defer closeBus() + sctx, cancel := step(ctx) + c, err := js.Consumer(sctx, stream, consumer) + cancel() + if err != nil { + if errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, fmt.Errorf("%s has no consumer %s (%v): nothing was removed", stream, consumer, err) + } + return nil, unanswered(err, fmt.Sprintf("the consumer %s of %s could not be read, so it was not removed", consumer, stream)) + } + info := c.CachedInfo() + was := map[string]any{ + "durable": info.Config.Durable, "filter_subject": info.Config.FilterSubject, "filter_subjects": info.Config.FilterSubjects, + "deliver_subject": info.Config.DeliverSubject, "pending": info.NumPending, "ack_pending": info.NumAckPending, + "redelivered": info.NumRedelivered, "created": info.Created, + } + sctx, cancel = step(ctx) + err = js.DeleteConsumer(sctx, stream, consumer) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("consumer %s of %s was not removed", consumer, stream)) + } + sctx, cancel = step(ctx) + _, err = js.Consumer(sctx, stream, consumer) + cancel() + if err == nil { + return nil, fmt.Errorf("consumer %s of %s is still there after the removal was accepted", consumer, stream) + } + out := ConsumerRemoved{Stream: stream, Consumer: consumer, Why: w, Was: was, Removed: true, At: time.Now().UTC()} + if err := said(consumerRemoved, out); err != nil { + return nil, fmt.Errorf("consumer %s of %s WAS removed (%s) but the audit event could not be said: %w", consumer, stream, w, err) + } + return out, nil +} + +// KeysRemoved is what nats_bucket_remove answers and says. +type KeysRemoved struct { + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Why string `json:"why"` + DryRun bool `json:"dry_run"` + Count int `json:"count"` + Keys []string `json:"keys"` + Cut bool `json:"keys_cut,omitempty"` + Removed int `json:"removed"` + At time.Time `json:"at"` +} + +// RemoveKeys removes the keys of a key-value bucket under a prefix: counted first, and only those. +func RemoveKeys(ctx context.Context, bus Bus, said Said, a map[string]any) (any, error) { + w, err := why(a) + if err != nil { + return nil, err + } + bucket, node, prefix := arg(a, "bucket"), arg(a, "node"), arg(a, "prefix") + if !name.MatchString(bucket) { + return nil, errors.New("bucket is required, a bucket's name as nats_buckets lists it: nothing was removed") + } + if (node == "") == (prefix == "") { + return nil, errors.New("give exactly one of node (that node's keys: those starting `.`) or prefix (keys starting so): " + + "a removal of a whole bucket is not this tool's; nothing was removed") + } + if node != "" { + if !name.MatchString(node) { + return nil, errors.New("node is one machine's name: nothing was removed") + } + prefix = node + "." + } + if !keyPrefix.MatchString(prefix) { + return nil, errors.New("prefix is key text without wildcards or spaces: nothing was removed") + } + dry, _ := a["dry_run"].(bool) + js, closeBus, err := bus(ctx) + if err != nil { + return nil, err + } + defer closeBus() + sctx, cancel := step(ctx) + kv, err := js.KeyValue(sctx, bucket) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("bucket %s could not be opened, nothing was removed", bucket)) + } + sctx, cancel = step(ctx) + stream, err := js.Stream(sctx, "KV_"+bucket) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("bucket %s has no stream to read its keys from", bucket)) + } + // The server narrows to the prefix's tokens when the prefix ends on a token; any other prefix is cut here. + filter := "$KV." + bucket + ".>" + if strings.HasSuffix(prefix, ".") { + filter = "$KV." + bucket + "." + prefix + ">" + } + sctx, cancel = step(ctx) + info, err := stream.Info(sctx, jetstream.WithSubjectFilter(filter)) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("the keys of %s could not be read, so none were removed", bucket)) + } + var keys []string + for subject := range info.State.Subjects { + if key := strings.TrimPrefix(subject, "$KV."+bucket+"."); strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + out := KeysRemoved{Bucket: bucket, Prefix: prefix, Why: w, DryRun: dry, Count: len(keys), Keys: keys, At: time.Now().UTC()} + if len(keys) > maxKeysShown { + out.Keys, out.Cut = keys[:maxKeysShown], true + } + if want, ok := a["expect"].(float64); ok && int(want) != len(keys) { + return nil, fmt.Errorf("%d keys of %s start with %q, not the %d expected: nothing was removed", len(keys), bucket, prefix, int(want)) + } + if dry || len(keys) == 0 { + return out, nil + } + for _, k := range keys { + sctx, cancel := step(ctx) + err := kv.Purge(sctx, k) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("removing %q from %s failed after %d of %d keys, the rest were left", k, bucket, out.Removed, len(keys))) + } + out.Removed++ + } + if err := said(bucketKeysRemoved, out); err != nil { + return nil, fmt.Errorf("%d keys of %s under %q WERE removed (%s) but the audit event could not be said: %w", out.Removed, bucket, prefix, w, err) + } + return out, nil +} + +// removalTools are the module's two tools that change the bus; every other tool here reads. +func removalTools(bus Bus, said Said) []stdio.Tool { + run := func(f func(context.Context, Bus, Said, map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(a map[string]any) (any, error) { return f(context.Background(), bus, said, a) } + } + return []stdio.Tool{ + {Name: "nats_consumer_remove", + Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. the consumer `controller` of DEAD_LETTER_NOTICES at the lease write (novox/hq ADR 0564 clause 5). Needs `why`, kept in the audit trail; refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", + Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")}, + Run: run(RemoveConsumer)}, + {Name: "nats_bucket_remove", + Description: "Removes (purges) keys from a key-value bucket: one node's keys (`node`, the keys starting `.`) or the keys under a `prefix`, e.g. a full acts bucket of the node-engine. Needs `why`, kept in the audit trail; never the whole bucket. Counts the keys first and lists them (`dry_run` only counts; `expect` refuses when the count differs); removes only those keys, and their history. See the buckets and their keys with nats_buckets. The bus refuses any bucket the module's credential is not granted.", + Input: map[string]any{"bucket": str("the bucket as nats_buckets lists it, e.g. mesh-host_acts"), "node": str("a node whose keys (`.`) go"), "prefix": str("key prefix instead of node"), + "why": str("why they are removed, for the audit trail (required)"), "dry_run": map[string]any{"type": "boolean", "description": "count and list the keys, remove none"}, "expect": num("the count of keys expected; refuses when it differs")}, + Run: run(RemoveKeys)}, + } +} diff --git a/modules/nats/cmd/nats-tools/removal_test.go b/modules/nats/cmd/nats-tools/removal_test.go new file mode 100644 index 00000000..2ed7799d --- /dev/null +++ b/modules/nats/cmd/nats-tools/removal_test.go @@ -0,0 +1,173 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// The two tools that change the bus, against a server shaped like the mesh's, on the credential the module's +// grant proposal gives them and nothing wider (issue 1952 N2, N3): the bus itself refuses what the grant leaves out, +// so this is also the proof that the proposed grant is enough and no wider than it needs to be. +func TestTheRemovalToolsChangeOnlyWhatTheyAreAskedAndTheGrantAllows(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "nats.conf") + if err := os.WriteFile(conf, []byte(fmt.Sprintf(`listen: "127.0.0.1:-1" +jetstream { store_dir: %q } +accounts { MESH { jetstream: enabled, users: [ + { user: "admin", password: "admin" }, + { user: "nats-tools", password: "tools", permissions: { + publish: { allow: [ + "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", + "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", + "$JS.API.STREAM.INFO.KV_mesh-host_acts", + "$KV.mesh-host_acts.>" ] }, + subscribe: { allow: ["_INBOX.nats-tools.>"] } } } +] } } +`, filepath.Join(dir, "js"))), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the server did not start") + } + defer s.Shutdown() + + answerWithin = 2 * time.Second + ctx := context.Background() + admin, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "admin")) + if err != nil { + t.Fatal(err) + } + defer admin.Close() + ajs, _ := jetstream.New(admin) + if _, err := ajs.CreateStream(ctx, jetstream.StreamConfig{Name: "DEAD_LETTER_NOTICES", Subjects: []string{"dl.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + for _, c := range []struct{ name, filter string }{{"controller", "dl.a"}, {"watchman", "dl.b"}} { + if _, err := ajs.CreateConsumer(ctx, "DEAD_LETTER_NOTICES", jetstream.ConsumerConfig{Durable: c.name, FilterSubject: c.filter, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } + acts, err := ajs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-host_acts"}) + if err != nil { + t.Fatal(err) + } + other, _ := ajs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "other_state"}) + for _, k := range []string{"g14.1", "g14.2", "g14.sub.3", "g14x.4", "ace.5"} { + acts.Put(ctx, k, []byte("v")) + } + other.Put(ctx, "g14.1", []byte("v")) + + bus := func(context.Context) (jetstream.JetStream, func(), error) { + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("nats-tools", "tools"), nats.CustomInboxPrefix("_INBOX.nats-tools")) + if err != nil { + return nil, nil, err + } + js, err := jetstream.New(nc) + return js, nc.Close, err + } + var said []string + say := func(event string, body any) error { said = append(said, event); return nil } + keys := func(kv jetstream.KeyValue) string { + ks, _ := kv.Keys(ctx) + sort.Strings(ks) + return strings.Join(ks, ",") + } + + // N2: the consumer goes, the stream and its other consumer stay; refused without why and for an unknown one. + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller"}); err == nil || !strings.Contains(err.Error(), "why is required") { + t.Fatalf("without why: %v", err) + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "controller"); err != nil { + t.Fatal("refused without why, but the consumer is gone") + } + // An unknown consumer: the granted name that does not exist is answered "not found"; an ungranted one is dropped. + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "nobody", "why": "x"}); err == nil || !strings.Contains(err.Error(), "not grant") && !strings.Contains(err.Error(), "no consumer") { + t.Fatalf("an unknown consumer: %v", err) + } + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "watchman", "why": "x"}); err == nil { + t.Fatal("a consumer outside the grant was removed") + } else if !strings.Contains(err.Error(), "grant") { + t.Fatalf("a consumer outside the grant: the refusal does not say so: %v", err) + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "watchman"); err != nil { + t.Fatalf("the grant did not hold, the watchman's consumer: %v", err) + } + got, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": "the lease write"}) + if err != nil { + t.Fatal(err) + } + if r := got.(ConsumerRemoved); !r.Removed || r.Was["filter_subject"] != "dl.a" || r.Why != "the lease write" { + t.Fatalf("%+v", r) + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "controller"); err == nil { + t.Fatal("the consumer is still there") + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "watchman"); err != nil { + t.Fatalf("the other consumer went too: %v", err) + } + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": "again"}); err == nil || !strings.Contains(err.Error(), "no consumer") { + t.Fatalf("a consumer already gone: %v", err) + } + + // N3: only the node's keys go; counted first; refused without why, for a whole bucket, and outside the grant. + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14"}); err == nil || !strings.Contains(err.Error(), "why is required") { + t.Fatalf("without why: %v", err) + } + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "why": "x"}); err == nil { + t.Fatal("a removal naming neither node nor prefix was not refused") + } + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "other_state", "node": "g14", "why": "x"}); err == nil { + t.Fatal("a bucket outside the grant was opened") + } + if k := keys(acts); k != "ace.5,g14.1,g14.2,g14.sub.3,g14x.4" { + t.Fatalf("keys after the refusals: %s", k) + } + dry, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14", "why": "look", "dry_run": true}) + if err != nil || dry.(KeysRemoved).Count != 3 || dry.(KeysRemoved).Removed != 0 { + t.Fatalf("dry run: %+v %v", dry, err) + } + if p, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "prefix": "g14.s", "why": "look", "dry_run": true}); err != nil || p.(KeysRemoved).Count != 1 { + t.Fatalf("a prefix that ends inside a token: %+v %v", p, err) + } + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14", "why": "x", "expect": 2.0}); err == nil { + t.Fatal("a count that differs from the expected was not refused") + } + got, err = RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14", "why": "the acts bucket is full", "expect": 3.0}) + if err != nil { + t.Fatal(err) + } + if r := got.(KeysRemoved); r.Removed != 3 || r.Count != 3 { + t.Fatalf("%+v", r) + } + if k := keys(acts); k != "ace.5,g14x.4" { + t.Fatalf("keys left: %s", k) + } + if k := keys(other); k != "g14.1" { + t.Fatalf("another bucket changed: %s", k) + } + if strings.Join(said, ",") != "consumer-removed,bucket-keys-removed" { + t.Fatalf("the audit events said: %v", said) + } +} diff --git a/modules/nats/cmd/nats-tools/tools.go b/modules/nats/cmd/nats-tools/tools.go index e725a070..88d5b8b9 100644 --- a/modules/nats/cmd/nats-tools/tools.go +++ b/modules/nats/cmd/nats-tools/tools.go @@ -351,6 +351,10 @@ func Streams(jsz []byte, stream string) (map[string]any, error) { if s.Config.MaxMsgsPerSubject > 0 { row["max_per_subject"] = s.Config.MaxMsgsPerSubject } + // The cap on the stream's size, so how full it is reads against it (-1 and 0 are no cap). + if s.Config.MaxBytes > 0 { + row["max_bytes"] = s.Config.MaxBytes + } if stream != "" { row["description"] = s.Config.Description var consumers []map[string]any diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index bd0e7d0f..2a2eca4a 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -1,12 +1,15 @@ package main import ( + "context" "encoding/json" "errors" "fmt" "strings" "testing" "time" + + "github.com/nats-io/nats.go/jetstream" ) const connz = `{"connections":[ @@ -19,7 +22,7 @@ const subsz = `{"num_subscriptions":3,"subscriptions_list":[ {"subject":"mesh.node.anchor.declare","msgs":1,"cid":1}]}` const jsz = `{"streams":2,"consumers":2,"messages":12,"bytes":300,"account_details":[{"stream_detail":[ - {"name":"EVENTS","config":{"subjects":["mesh.mod.*.event.>"],"retention":"limits","max_age":604800000000000,"max_msgs_per_subject":10000}, + {"name":"EVENTS","config":{"subjects":["mesh.mod.*.event.>"],"retention":"limits","max_age":604800000000000,"max_msgs_per_subject":10000,"max_bytes":1048576}, "state":{"messages":10,"bytes":200,"num_subjects":3,"consumer_count":2,"last_ts":"2026-10-04T10:00:00Z"}, "consumer_detail":[ {"name":"anchor_audit","config":{"filter_subject":"mesh.mod.*.event.>"},"num_pending":5,"num_ack_pending":1,"num_redelivered":2,"delivered":{"last_active":"2026-10-04T10:00:00Z"}}, @@ -77,7 +80,7 @@ func TestStreamsBacklogAndBuckets(t *testing.T) { } one, _ := Streams([]byte(jsz), "EVENTS") row := one["detail"].([]map[string]any)[0] - if row["max_age_hours"].(int64) != 168 || len(row["consumer_detail"].([]map[string]any)) != 2 { + if row["max_age_hours"].(int64) != 168 || row["max_bytes"].(int64) != 1048576 || len(row["consumer_detail"].([]map[string]any)) != 2 { t.Fatalf("%v", row) } if _, err := Streams([]byte(jsz), "NOPE"); err == nil { @@ -208,3 +211,47 @@ func TestAGrantIsInForceOnlyWhenTheServerReloadedAfterTheListWasWritten(t *testi t.Error("a write time that is not one was read") } } + +// **The tools that change the bus are two, each refuses before it reaches the bus, and every other tool reads** +// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them. +func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { + bus := func(context.Context) (jetstream.JetStream, func(), error) { + t.Fatal("the bus was opened for a call that must be refused") + return nil, nil, nil + } + said := func(string, any) error { t.Fatal("an audit event was said for a refused call"); return nil } + for _, c := range []struct { + tool string + args map[string]any + want string + }{ + {"nats_consumer_remove", map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller"}, "why is required"}, + {"nats_consumer_remove", map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": " "}, "why is required"}, + {"nats_consumer_remove", map[string]any{"stream": "A.B", "consumer": "c", "why": "x"}, "stream and consumer are required"}, + {"nats_consumer_remove", map[string]any{"stream": "S", "consumer": ">", "why": "x"}, "stream and consumer are required"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "node": "g14"}, "why is required"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "why": "x"}, "exactly one of node"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "node": "g14", "prefix": "g", "why": "x"}, "exactly one of node"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "prefix": "g*", "why": "x"}, "without wildcards"}, + {"nats_bucket_remove", map[string]any{"bucket": "", "node": "g14", "why": "x"}, "bucket is required"}, + } { + var run func(map[string]any) (any, error) + for _, tl := range removalTools(bus, said) { + if tl.Name == c.tool { + run = tl.Run + } + } + if run == nil { + t.Fatalf("no tool %s", c.tool) + } + if _, err := run(c.args); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s %v: %v, want a refusal saying %q", c.tool, c.args, err, c.want) + } + } + // The read tools stay read tools: none of the first ten takes a `why`, and the list is these ten. + for _, tl := range tools(nil, nil, nil) { + if _, ok := tl.Input["why"]; ok || strings.Contains(tl.Name, "remove") { + t.Errorf("%s changes the bus; only the removal tools may", tl.Name) + } + } +} diff --git a/modules/nats/go.mod b/modules/nats/go.mod index 9f7e38c0..8e261655 100644 --- a/modules/nats/go.mod +++ b/modules/nats/go.mod @@ -1,12 +1,12 @@ module nats-tools -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.7 - github.com/klauspost/compress v1.18.5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/klauspost/compress v1.20.0 github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nats.go v1.54.0 ) require ( @@ -14,9 +14,9 @@ require ( github.com/google/go-tpm v0.9.8 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/nats/go.sum b/modules/nats/go.sum index b28aa2f0..a2b8e018 100644 --- a/modules/nats/go.sum +++ b/modules/nats/go.sum @@ -1,27 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/nats/internal/busconn/busconn.go b/modules/nats/internal/busconn/busconn.go new file mode 100644 index 00000000..09ada3c3 --- /dev/null +++ b/modules/nats/internal/busconn/busconn.go @@ -0,0 +1,106 @@ +// Package busconn dials the bus as a credential the mesh sealed to a module: the one place the module's +// programs learn how (the snapshot, and the tools that change the bus). +package busconn + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/nats-io/nats.go" +) + +// Credential is what the mesh seals to a module so it can reach the bus (mesh-controller +// cmd/mesh-controller issueWith): where, the server's certificate by fingerprint, who, and the +// password. The address is not used here — this runs beside the server, and the server is reached on +// loopback — but the fingerprint is: loopback or not, the certificate is the mesh's own. +type Credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + User string `json:"user"` + Password string `json:"password"` +} + +func Read(path string) (Credential, error) { + var raw []byte + var err error + if path == "-" { + raw, err = io.ReadAll(io.LimitReader(os.Stdin, 1<<20)) + } else { + raw, err = os.ReadFile(path) + } + if err != nil { + return Credential{}, fmt.Errorf("cannot read the credential: %w", err) + } + var c Credential + if err := json.Unmarshal(raw, &c); err != nil { + return Credential{}, errors.New("the credential is not the JSON the mesh seals to a module (url, fingerprint, user, password)") + } + if c.User == "" || c.Password == "" { + return Credential{}, errors.New("the credential names no user or password, so the bus would refuse it") + } + return c, nil +} + +// Connect dials the bus as the credential's user. A `tls://` server is held to the credential's +// fingerprint and nothing else — the mesh's bus presents a certificate of its own, in no trust store, +// so a name check could only fail or be skipped. A plain `nats://` server is accepted only on +// loopback, and is what a restore's own private server is. +func Connect(server string, c Credential, name string) (*nats.Conn, error) { + opts := []nats.Option{ + nats.UserInfo(c.User, c.Password), + nats.Name(name), + // Answers come to the user's own inbox and no other: the bus grants every user its own + // `_INBOX..>` and nothing wider (novox/hq design 25 §4). + nats.CustomInboxPrefix("_INBOX." + c.User), + nats.Timeout(10 * time.Second), + // One attempt: a program that cannot reach the bus fails, loudly, rather than waiting for it. + nats.NoReconnect(), + } + switch { + case strings.HasPrefix(server, "tls://"): + if c.Fingerprint == "" { + return nil, errors.New("the credential carries no fingerprint, so the bus's certificate cannot be checked") + } + opts = append(opts, nats.Secure(PinnedTo(c.Fingerprint))) + case strings.HasPrefix(server, "nats://127.0.0.1:"), strings.HasPrefix(server, "nats://localhost:"): + if c.Fingerprint != "" { + // The mesh's bus requires TLS; a plain connection to it would be refused anyway. + opts = append(opts, nats.Secure(PinnedTo(c.Fingerprint))) + } + default: + return nil, fmt.Errorf("%s: the bus is reached over tls:// (or plain nats:// on loopback, for a restore's own server)", server) + } + nc, err := nats.Connect(server, opts...) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", server, c.User, err) + } + return nc, nil +} + +// PinnedTo accepts exactly the certificate with this SHA-256 — the same pin every machine holds +// (mesh-controller internal/broker PinnedToFingerprint). +func PinnedTo(want string) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter + MinVersion: tls.VersionTLS12, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("the bus presented no certificate") + } + sum := sha256.Sum256(rawCerts[0]) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != want { + return fmt.Errorf("the bus presented a certificate this mesh does not know (%s)", got) + } + return nil + }, + } +} diff --git a/modules/nats/module.json b/modules/nats/module.json index 106aac92..d2d02efe 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -8,7 +8,10 @@ "provides": [ { "name": "mesh-bus", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "claims": [ @@ -19,12 +22,16 @@ ], "bus-users": "/var/lib/nats-module/conf/accounts.conf", "own-secrets": { - "broker": "${dir:mesh-state}/broker" + "broker": "${dir:mesh-state}/broker", + "broker-nats-tools": "${dir:tools-secret}/broker-nats-tools" }, "capabilities": [ "container-runtime" ], - "emits": [], + "emits": [ + "consumer-removed", + "bucket-keys-removed" + ], "consumes": [], "listens": [ { @@ -45,7 +52,9 @@ "nats_buckets", "nats_users", "nats_user_can", - "nats_snapshot_verify" + "nats_snapshot_verify", + "nats_consumer_remove", + "nats_bucket_remove" ], "guards": [ 8222 @@ -86,6 +95,13 @@ "mode": "0700", "unknown": "program" }, + { + "id": "tools-secret", + "type": "directory", + "path": "/var/lib/mesh/nats-tools", + "mode": "0711", + "unknown": "program" + }, { "id": "snapshots", "type": "directory", @@ -190,11 +206,18 @@ "loads": [ "nats-tools" ], + "bus": { + "removes": [ + "DEAD_LETTER_NOTICES.controller", + "mesh-host_acts" + ] + }, "env": { "MESH_NATS_MONITOR": "http://127.0.0.1:8222", "MESH_NATS_CONTAINER": "mesh-broker-nats", "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf", - "MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}" + "MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}", + "NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools" } }, { diff --git a/modules/nfs-server/.gitignore b/modules/nfs-server/.gitignore new file mode 100644 index 00000000..3cbd8705 --- /dev/null +++ b/modules/nfs-server/.gitignore @@ -0,0 +1 @@ +nfs-server diff --git a/modules/nfs-server/cmd/nfs-server/backup.go b/modules/nfs-server/cmd/nfs-server/backup.go index 8bf3d6fd..99c7ba59 100644 --- a/modules/nfs-server/cmd/nfs-server/backup.go +++ b/modules/nfs-server/cmd/nfs-server/backup.go @@ -14,12 +14,12 @@ package main // been read, what is exported stays exported and the status says the list is not known: the bus being // slow is no reason to take the library away, and the look below still guards against a repository. // -// The parsing of mesh-controller.data's answer is a small copy of the mounts module's (DataFrom). Shared -// code belongs in mesh-sdk: moving both there is mesh-sdk issue 13. import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path" "sort" @@ -46,68 +46,46 @@ type BackupFromBus struct { Error string `json:"error,omitempty"` } -// BackupFrom reads the holder of node-backup on one machine from mesh-controller.seats' answer, and the -// paths that module keeps data in on that machine from mesh-controller.data's. A machine whose backup seat -// nobody holds, and an answer with no data list, are errors: never an empty list read as "nothing to fear". -func BackupFrom(machine string, seats, data json.RawMessage) (string, []string, error) { - var s struct { - Answer *struct { - Seats []struct { - Seat string `json:"seat"` - Holders []struct { - Module string `json:"module"` - Node string `json:"node"` - } `json:"holders"` - } `json:"seats"` - } `json:"answer"` - } - if err := json.Unmarshal(seats, &s); err != nil || s.Answer == nil { - return "", nil, fmt.Errorf("the controller's seats answer could not be read: %v", err) +// BackupFrom reads the holder of node-backup on one machine from the placement seat's state of who holds which seat +// (novox/hq ADR 0571 clause 4), and the paths that module keeps data in on that machine from disk-load's readings +// (clause 3). A machine whose backup seat nobody holds, and no readings at all, are errors: never an empty list read as +// "nothing to fear". +func BackupFrom(machine string, keys map[string][]byte, readings []health.Reading) (string, []string, error) { + holders, err := delivery.HoldersOf(keys, BackupSeat, machine) + if err != nil { + return "", nil, err } holder := "" - for _, seat := range s.Answer.Seats { - if seat.Seat != BackupSeat { - continue - } - for _, h := range seat.Holders { - if h.Node == machine { - holder = h.Module - } + for _, h := range holders { + if h.Node == machine { + holder = h.Module } } if holder == "" { return "", nil, fmt.Errorf("no module holds %s on %s", BackupSeat, machine) } - var d struct { - Answer *struct { - Data *[]struct { - Machine string `json:"machine"` - Module string `json:"module"` - Path string `json:"path"` - } `json:"data"` - } `json:"answer"` - } - if err := json.Unmarshal(data, &d); err != nil || d.Answer == nil || d.Answer.Data == nil { - return "", nil, fmt.Errorf("the controller's data answer holds no data list (%v)", err) + if len(readings) == 0 { + return "", nil, fmt.Errorf("disk-load keeps no reading of this machine's data yet") } var paths []string - for _, item := range *d.Answer.Data { - if item.Module == holder && (item.Machine == "" || item.Machine == machine) && path.IsAbs(item.Path) { - paths = append(paths, path.Clean(item.Path)) + for _, r := range readings { + if r.Module == holder && (r.Node == "" || r.Node == machine) && path.IsAbs(r.Path) { + paths = append(paths, path.Clean(r.Path)) } } sort.Strings(paths) return holder, paths, nil } -// TakeBackup asks the controller over the bus, as the module, through the runtime. +// TakeBackup reads the bus, as the module, through the runtime. func (m *Machine) TakeBackup(machine string) { - m.TakeBackupWith(machine, askBus) + m.TakeBackupWith(machine, readingsOf, seatHolderKeys) } -// TakeBackupWith is TakeBackup with the ask given, for tests. A failed ask keeps the last holder and +// TakeBackupWith is TakeBackup with the ask and the reading of the seat-holders state given, for tests. A failed ask keeps the last holder and // paths and adds why. -func (m *Machine) TakeBackupWith(machine string, ask func(key string, body any) (json.RawMessage, error)) { +func (m *Machine) TakeBackupWith(machine string, readings func(node string) ([]health.Reading, error), + holderKeys func() (map[string][]byte, error)) { var last BackupFromBus if text, err := m.Read(BackupFile); err == nil { _ = json.Unmarshal([]byte(text), &last) @@ -115,10 +93,10 @@ func (m *Machine) TakeBackupWith(machine string, ask func(key string, body any) next := last next.Looked = m.Now().UTC().Format(time.RFC3339) next.Error = "" - seats, err := ask("seat:mesh-controller.seats", map[string]any{}) - var data json.RawMessage + seats, err := holderKeys() + var data []health.Reading if err == nil { - data, err = ask("seat:mesh-controller.data", map[string]any{"machine": machine}) + data, err = readings(machine) } if err == nil { var holder string diff --git a/modules/nfs-server/cmd/nfs-server/backup_test.go b/modules/nfs-server/cmd/nfs-server/backup_test.go index 7ae3298b..dd4da159 100644 --- a/modules/nfs-server/cmd/nfs-server/backup_test.go +++ b/modules/nfs-server/cmd/nfs-server/backup_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "strings" "testing" "time" @@ -81,8 +83,8 @@ func TestWithoutTheBackupListTheExportsStayAndItIsSaid(t *testing.T) { } func TestBackupFromTheSeatsAndTheData(t *testing.T) { - seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","scope":"node","holders":[{"module":"restic","node":"ace"},{"module":"borg","node":"shanks"}]}]},"ok":true}`) - data := json.RawMessage(`{"answer":{"data":[{"machine":"ace","module":"restic","item":"repository","path":"/storage/restic"},{"machine":"ace","module":"restic","item":"state","path":"/var/lib/restic"},{"machine":"ace","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`) + seats := backupHolders("ace", "restic") + data := readingsOfPaths("ace", map[string]string{"restic/repository": "/storage/restic", "restic/state": "/var/lib/restic", "plex/data": "/mnt/plex/data"}) holder, paths, err := BackupFrom("ace", seats, data) if err != nil || holder != "restic" || strings.Join(paths, " ") != "/storage/restic /var/lib/restic" { t.Errorf("got %q %v %v", holder, paths, err) @@ -90,8 +92,8 @@ func TestBackupFromTheSeatsAndTheData(t *testing.T) { if _, _, err := BackupFrom("g14", seats, data); err == nil { t.Errorf("a machine whose backup seat nobody holds is not said") } - if _, _, err := BackupFrom("ace", seats, json.RawMessage(`{"ok":true}`)); err == nil { - t.Errorf("an answer with no data list reads as an empty list") + if _, _, err := BackupFrom("ace", seats, nil); err == nil { + t.Errorf("no readings read as an empty list") } } @@ -99,13 +101,13 @@ func TestBackupFromTheSeatsAndTheData(t *testing.T) { func TestAFailedAskKeepsTheLastBackupPaths(t *testing.T) { f := newFake(t) m := f.machine() - m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) { + m.TakeBackupWith("ace", func(string) ([]health.Reading, error) { return nil, errors.New("no responders") - }) + }, noHolders) f.backup("restic", "/storage/restic") - m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) { + m.TakeBackupWith("ace", func(string) ([]health.Reading, error) { return nil, errors.New("timeout") - }) + }, func() (map[string][]byte, error) { return backupHolders("ace", "restic"), nil }) var b BackupFromBus if err := json.Unmarshal([]byte(f.read(BackupFile)), &b); err != nil { t.Fatal(err) @@ -152,15 +154,13 @@ func TestAStaleOrUnknownBackupListIsSaidWithItsAge(t *testing.T) { t.Errorf("an unknown backup list is not a warning in the health: %+v", h) } - ok := func(key string, body any) (json.RawMessage, error) { - if strings.Contains(key, "seats") { - return json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","holders":[{"module":"restic","node":"hs"}]}]}}`), nil - } - return json.RawMessage(`{"answer":{"data":[{"machine":"hs","module":"restic","path":"/srv/restic"}]}}`), nil + ok := func(string) ([]health.Reading, error) { + return readingsOfPaths("hs", map[string]string{"restic/repository": "/srv/restic"}), nil } - m.TakeBackupWith("hs", ok) + m.TakeBackupWith("hs", ok, func() (map[string][]byte, error) { return backupHolders("hs", "restic"), nil }) now = now.Add(2 * time.Hour) - m.TakeBackupWith("hs", func(string, any) (json.RawMessage, error) { return nil, errors.New("the bus did not answer") }) + m.TakeBackupWith("hs", func(string) ([]health.Reading, error) { return nil, errors.New("the bus did not answer") }, + func() (map[string][]byte, error) { return backupHolders("hs", "restic"), nil }) s := m.Pass(context.Background(), nil) said := strings.Join(append(append([]string{}, s.Said...), s.Warnings...), "\n") if !strings.Contains(said, "2h0m0s") || !strings.Contains(said, "the bus did not answer") { @@ -171,3 +171,21 @@ func TestAStaleOrUnknownBackupListIsSaidWithItsAge(t *testing.T) { t.Errorf("the health does not say the backup list's age: %+v", h) } } + +// backupHolders is the placement seat's state with one holder of node-backup on one node. +func backupHolders(node, module string) map[string][]byte { + return map[string][]byte{"node-backup." + node: []byte(`{"v":1,"seat":"node-backup","scope":"node","node":"` + node + + `","holders":[{"module":"` + module + `","node":"` + node + `"}]}`)} +} + +func noHolders() (map[string][]byte, error) { return nil, errors.New("the state is not read yet") } + +// readingsOfPaths is disk-load's readings of one node's items, by "/", each at its path. +func readingsOfPaths(node string, items map[string]string) []health.Reading { + var out []health.Reading + for k, p := range items { + module, item, _ := strings.Cut(k, "/") + out = append(out, health.Reading{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module, ID: item, Path: p}}) + } + return out +} diff --git a/modules/nfs-server/cmd/nfs-server/guards_test.go b/modules/nfs-server/cmd/nfs-server/guards_test.go index add729ff..457af6f6 100644 --- a/modules/nfs-server/cmd/nfs-server/guards_test.go +++ b/modules/nfs-server/cmd/nfs-server/guards_test.go @@ -4,7 +4,7 @@ package main import ( "context" - "encoding/json" + "git.novox.be/novox/mesh-sdk/go/health" "strings" "testing" ) @@ -63,12 +63,10 @@ func TestAFailedReloadIsTriedAgainWhenNothingElseAsks(t *testing.T) { } } -func TestBackupFromRefusesAnAnswerWithoutADataList(t *testing.T) { - seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","holders":[{"module":"restic","node":"ace"}]}]}}`) - for _, data := range []string{`{"answer":{}}`, `{"answer":{"data":null}}`} { - if _, paths, err := BackupFrom("ace", seats, json.RawMessage(data)); err == nil { - t.Errorf("%s was read as the empty list %v: never \"nothing to fear\"", data, paths) - } +func TestBackupFromRefusesNoReadingsAtAll(t *testing.T) { + seats := backupHolders("ace", "restic") + if _, paths, err := BackupFrom("ace", seats, []health.Reading{}); err == nil { + t.Errorf("no readings were read as the empty list %v: never \"nothing to fear\"", paths) } } diff --git a/modules/nfs-server/cmd/nfs-server/main.go b/modules/nfs-server/cmd/nfs-server/main.go index 3406d5a8..6edda7de 100644 --- a/modules/nfs-server/cmd/nfs-server/main.go +++ b/modules/nfs-server/cmd/nfs-server/main.go @@ -16,10 +16,13 @@ import ( "os" "os/signal" "sort" + "strings" "syscall" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // ExportsState is the state the exports are kept in on the bus; MountsState the `mounts` modules' state, @@ -88,9 +91,50 @@ func ShareExports(m *Machine) { } } -// askBus calls a seat's verb over the bus as this module, through the runtime. -func askBus(key string, body any) (json.RawMessage, error) { - return stdio.Ask(key, body) +// readingsOf reads disk-load's readings of one machine's data items, every key `..`. +func readingsOf(node string) ([]health.Reading, error) { + state := stdio.State(health.ReadingsRead) + names, err := state.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range names { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := state.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil +} + +// seatHolderKeys reads the placement seat's state of who holds which seat, every key. +func seatHolderKeys() (map[string][]byte, error) { + state := stdio.State(delivery.SeatHoldersRead) + names, err := state.Keys() + if err != nil { + return nil, err + } + keys := map[string][]byte{} + for _, k := range names { + e, err := state.Get(k) + if err != nil { + return nil, err + } + if e != nil { + keys[k] = e.Value + } + } + return keys, nil } // TakeWants reads every node's `mounts` state from the bus and writes what is asked of this machine to diff --git a/modules/nfs-server/cmd/nfs-server/manifest_test.go b/modules/nfs-server/cmd/nfs-server/manifest_test.go index d43d5776..ae048eb2 100644 --- a/modules/nfs-server/cmd/nfs-server/manifest_test.go +++ b/modules/nfs-server/cmd/nfs-server/manifest_test.go @@ -2,6 +2,8 @@ package main import ( "encoding/json" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "os" "regexp" "sort" @@ -137,7 +139,7 @@ func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing. if err := json.Unmarshal([]byte(raw), &full); err != nil { t.Fatal(err) } - if len(full.Reads) != 1 || full.Reads[0] != MountsState { + if len(full.Reads) != 3 || full.Reads[0] != health.ReadingsRead || full.Reads[1] != delivery.SeatHoldersRead || full.Reads[2] != MountsState { t.Errorf("reads: %v", full.Reads) } roster := full.Facts["machines"] diff --git a/modules/nfs-server/go.mod b/modules/nfs-server/go.mod index 6e122025..ffeb87e5 100644 --- a/modules/nfs-server/go.mod +++ b/modules/nfs-server/go.mod @@ -1,5 +1,5 @@ module nfsserver -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/nfs-server/go.sum b/modules/nfs-server/go.sum index b474419e..09ad064e 100644 --- a/modules/nfs-server/go.sum +++ b/modules/nfs-server/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index f9994d16..c7082176 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -13,7 +13,10 @@ { "name": "nfs-share", "scope": "mesh", - "identity": false + "identity": false, + "credential": { + "form": "none" + } } ], "data": { @@ -45,26 +48,24 @@ } ], "reads": [ + "disk-load.readings", + "mesh-delivery-assigner.seat-holders", "mounts.shares" ], - "invokes": [ - "seat:mesh-controller.seats", - "seat:mesh-controller.data" - ], - "config": { - "shares": { - "type": "string", - "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", - "trusted": true, - "required": true - }, - "grants": { - "type": "string", - "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", - "trusted": true, - "default": "none" - } + "config": { + "shares": { + "type": "string", + "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", + "trusted": true, + "required": true }, + "grants": { + "type": "string", + "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", + "trusted": true, + "default": "none" + } + }, "facts": { "machines": { "path": "/etc/nfs-server/machines", diff --git a/modules/nodejs/module.json b/modules/nodejs/module.json index c3573444..2f4a049a 100644 --- a/modules/nodejs/module.json +++ b/modules/nodejs/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "node-toolchain", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "resources": [ diff --git a/modules/ollama/module.json b/modules/ollama/module.json index 2d5fb48c..312ef6bf 100644 --- a/modules/ollama/module.json +++ b/modules/ollama/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "model-access", - "scope": "node" + "scope": "node", + "credential": { + "form": "none" + } } ], "listens": [ diff --git a/modules/postgres/cmd/postgres-provider/harness_test.go b/modules/postgres/cmd/postgres-provider/harness_test.go index 26b0246c..7c04bdba 100644 --- a/modules/postgres/cmd/postgres-provider/harness_test.go +++ b/modules/postgres/cmd/postgres-provider/harness_test.go @@ -120,8 +120,8 @@ func TestProvisionerCreatesTheDatabaseThenItsExtensions(t *testing.T) { if err := a.Retire(ctx, "mesh_ace_letta", nil, "test", time.Now()); err != nil { t.Fatal(err) } - noDrop(t, f.statements()) - if !has(f.statements(), `NOLOGIN`) { + // On the operator's yes the login goes and the database stays (ADR 0570 clause 5). + if !has(f.statements(), `^DROP ROLE "mesh_ace_letta"$`) || has(f.statements(), `DROP DATABASE`) { t.Fatal(f.statements()) } } diff --git a/modules/postgres/cmd/postgres-provider/live_test.go b/modules/postgres/cmd/postgres-provider/live_test.go index 38f91ef5..5e847973 100644 --- a/modules/postgres/cmd/postgres-provider/live_test.go +++ b/modules/postgres/cmd/postgres-provider/live_test.go @@ -86,6 +86,14 @@ func TestLive(t *testing.T) { if err != nil || len(r.Rows) != 1 || cell(r.Rows[0], 0) != "1" { t.Fatal("retiring lost the data:", r, err) } + // The login is removed, its database kept by its keeper (ADR 0570 clause 5). + if has, _ := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(p.As)); has { + t.Fatal("the retired login is still there") + } + if has, _ := c.exists(ctx, "SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE d.datname = "+ + Literal(p.As)+" AND r.rolname = "+Literal(p.As+"~retired")); !has { + t.Fatal("the database is not its keeper's") + } inv, err := a.Inventory(ctx) if err != nil { t.Fatal(err) diff --git a/modules/postgres/cmd/postgres-provider/provisioner.go b/modules/postgres/cmd/postgres-provider/provisioner.go index a0d4ed06..b9bbb02d 100644 --- a/modules/postgres/cmd/postgres-provider/provisioner.go +++ b/modules/postgres/cmd/postgres-provider/provisioner.go @@ -35,10 +35,14 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error { if err := a.pg.CreateDatabaseAndRole(ctx, database, p.As, p.Password); err != nil { return err } + // A login retired and asked for again takes back what its keeper kept (retire_pg.go). + if err := a.pg.Reclaim(ctx, database, p.As); err != nil { + return err + } if err := a.pg.EnsureExtensions(ctx, database, extensions); err != nil { return err } - // The active mark: a retired consumer asked for again loses its mark to delete here, its LOGIN + // The active mark: a login locked before ADR 0570 and asked for again loses its mark to delete here, its LOGIN // already set again by the role statement above (novox/hq ADR 0230). if err := a.pg.MarkActive(ctx, p.As, p.Consumer); err != nil { return err @@ -47,11 +51,10 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error { return nil } -// Retire locks the login, never drops (novox/hq issue 241, ADR 0230): the database is kept under its -// own name, the role marked retired with when and why (retire_pg.go). On 2026-10-04 a misread -// contributions file withdrew every consumer at once, and dropping made that a loss of seven databases. -// Deleting is `cleanup delete`, a person's act; taking a database out of service by hand is -// postgres_retire_database, which renames rather than drops. +// Retire removes the login on the operator's yes (novox/hq ADR 0570 clause 5, ADR 0114 §2): what it owns is handed +// to its keeper, its sessions end and the login is dropped; the database and its data are kept (retire_pg.go). +// Deleting is `cleanup delete`, a person's act; taking a database out of service by hand is postgres_retire_database, +// which renames rather than drops. func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error { if err := a.pg.RetireRole(ctx, as, why, at); err != nil { return err diff --git a/modules/postgres/cmd/postgres-provider/retire_pg.go b/modules/postgres/cmd/postgres-provider/retire_pg.go index e07a5640..24e911c8 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg.go @@ -1,20 +1,22 @@ package main -// What retired means in postgres (novox/hq ADR 0230). +// What retired means in postgres (novox/hq ADR 0570 clause 5, ADR 0114 §2 and §9; ADR 0230 for the delete). // -// **Retired is the login locked, the database kept, and the role marked.** `ALTER ROLE … NOLOGIN` — -// the consumer's login can no longer connect, as itself, to anything — and its open sessions are -// ended. The database, its owner, its grants and every byte in it stay exactly as they were: CONNECT is -// not revoked and the database still allows connections, because the backup contribution dumps every -// database and a retired one is still worth backing up, and because re-enabling must give it back as it -// was. The mark is the role's comment, a JSON object the mesh owns: +// **Retired is the login removed, the database kept under a keeper role, and the keeper marked.** Retiring happens +// only on the operator's yes to a retiring provision (the SDK's moved path, Gone). What the login owns, its database +// first, is handed to its keeper, a role of its own that cannot log in (`~retired`, KeeperOf), its open +// sessions are ended, and the login is dropped: nothing can connect as it any more, and no credential of it is left. +// The database, its grants to others and every byte in it stay: the backup contribution dumps every database and a +// retired one is still worth backing up. The mark is the keeper's comment, a JSON object the mesh owns: // -// {"mesh":"consumer","node":"ace"} active -// {"mesh":"consumer","node":"ace","retired":"2026-10-06T…Z","why":"…"} retired +// {"mesh":"consumer","node":"ace"} an active login +// {"mesh":"consumer","node":"ace","retired":"2026-10-06T…Z","why":"…","login":"…"} a keeper // -// Asked for again, the ordinary create sets LOGIN and the password again and writes the active mark. -// Deleted — only through `cleanup delete` — the database is dropped, then the role, unless it still owns -// another database (a set-aside copy), which is said and kept. +// Asked for again, the ordinary create makes the login again and its keeper hands everything back (Reclaim) and +// goes. Deleted — only through `cleanup delete` — the database is dropped, then the keeper. +// +// **A login retired before this record** was only locked (`NOLOGIN`, its own role marked retired); it is listed and +// deleted as before, and create sets its LOGIN again. // // **What the mesh made is recognised by its mark, or by its shape before the mark existed**: a role // valid until 'infinity' (only the mesh's role statement says that) that owns a database of its own @@ -41,6 +43,17 @@ type roleMark struct { Node string `json:"node,omitempty"` Retired string `json:"retired,omitempty"` Why string `json:"why,omitempty"` + // Login is, on a keeper, the login it keeps for. + Login string `json:"login,omitempty"` +} + +// KeeperOf is the role a retired login's database and objects are handed to: the login's name with `~retired`. +func KeeperOf(login string) (string, error) { + k := login + "~retired" + if len(k) > 63 { + return "", fmt.Errorf("%s is too long a login for a keeper role (%s is over postgres's 63 bytes)", login, k) + } + return k, nil } func readMark(comment string) (roleMark, bool) { @@ -64,8 +77,9 @@ func (c *Client) MarkActive(ctx context.Context, role, node string) error { return err } -// RetireRole locks the login, ends its sessions and marks it retired with when and why. Its database -// is not touched. A role that does not exist has nothing to retire. +// RetireRole removes a login on the operator's yes (ADR 0570 clause 5): what it owns is handed to its keeper, marked +// with when and why, its sessions end and the login is dropped. The database and its data are kept. A login that does +// not exist has nothing to retire. func (c *Client) RetireRole(ctx context.Context, role, why string, at time.Time) error { r, err := c.Query(ctx, "", "SELECT coalesce(shobj_description(oid, 'pg_authid'), '') FROM pg_roles WHERE rolname = "+ Literal(role)) @@ -76,10 +90,66 @@ func (c *Client) RetireRole(ctx context.Context, role, why string, at time.Time) return nil } prev, _ := readMark(cell(r.Rows[0], 0)) + keeper, err := KeeperOf(role) + if err != nil { + return err + } + has, err := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(keeper)) + if err != nil { + return err + } + if !has { + if _, err := c.Query(ctx, "", fmt.Sprintf("CREATE ROLE %s NOLOGIN", Ident(keeper))); err != nil { + return err + } + } + if _, err := c.Query(ctx, "", MarkStatement(keeper, roleMark{Node: prev.Node, Retired: at.UTC().Format(time.RFC3339), + Why: why, Login: role})); err != nil { + return err + } + // No new session, and none left open. if err := c.LockRole(ctx, role); err != nil { return err } - _, err = c.Query(ctx, "", MarkStatement(role, roleMark{Node: prev.Node, Retired: at.UTC().Format(time.RFC3339), Why: why})) + // In its own database, where its objects are; the database itself is a shared object, handed with them. + in := "" + if own, err := c.exists(ctx, "SELECT 1 FROM pg_database WHERE datname = "+Literal(role)); err != nil { + return err + } else if own { + in = role + } + for _, sql := range []string{ + fmt.Sprintf("REASSIGN OWNED BY %s TO %s", Ident(role), Ident(keeper)), + fmt.Sprintf("DROP OWNED BY %s", Ident(role)), + } { + if _, err := c.Query(ctx, in, sql); err != nil { + return err + } + } + _, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(role))) + return err +} + +// Reclaim gives a login made again what its keeper kept (ADR 0570 clause 5: a provision needed again): the keeper's +// database and objects are handed back, and the keeper goes. A login with no keeper has nothing to reclaim. +func (c *Client) Reclaim(ctx context.Context, database, role string) error { + keeper, err := KeeperOf(role) + if err != nil { + return err + } + has, err := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(keeper)) + if err != nil || !has { + return err + } + for _, sql := range []string{ + fmt.Sprintf("REASSIGN OWNED BY %s TO %s", Ident(keeper), Ident(role)), + fmt.Sprintf("DROP OWNED BY %s", Ident(keeper)), + } { + if _, err := c.Query(ctx, database, sql); err != nil { + return err + } + } + _, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(keeper))) return err } @@ -89,7 +159,8 @@ var setAside = regexp.MustCompile(`_deleted_(\d{8})$`) // of its own database. const InventoryStatement = `SELECT r.rolname, r.rolcanlogin, coalesce(shobj_description(r.oid, 'pg_authid'), '') AS mark, coalesce(r.rolvaliduntil = 'infinity', false) AS mesh_shaped, - (SELECT pg_database_size(d.datname) FROM pg_database d WHERE d.datname = r.rolname AND d.datdba = r.oid) AS size + (SELECT pg_database_size(d.datname) FROM pg_database d WHERE d.datdba = r.oid + AND (d.datname = r.rolname OR d.datname || '~retired' = r.rolname)) AS size FROM pg_roles r WHERE r.rolname !~ '^pg_' AND NOT r.rolsuper AND r.rolname <> ` + "'" + Reader + "'" + ` ORDER BY r.rolname` @@ -118,6 +189,9 @@ func (c *Client) Inventory(ctx context.Context) (harness.Inventory, error) { continue } at, _ := time.Parse(time.RFC3339, m.Retired) + if m.Login != "" { + name = m.Login // a keeper: listed under the login it keeps for + } inv.Retired = append(inv.Retired, harness.Retired{Consumer: name, Node: m.Node, RetiredAt: at, Why: m.Why, SizeBytes: sizeOf(size), Kind: harness.KindConsumer}) } @@ -160,7 +234,19 @@ func (c *Client) DeleteRetired(ctx context.Context, r harness.Retired) (int64, e return freed, err } if len(row.Rows) == 0 { - return freed, nil + // Retired by removing the login: its keeper goes with the database it kept. + keeper, err := KeeperOf(r.Consumer) + if err != nil { + return freed, err + } + if has, err := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(keeper)); err != nil || !has { + return freed, err + } + if _, err := c.Query(ctx, "", fmt.Sprintf("DROP OWNED BY %s", Ident(keeper))); err != nil { + return freed, err + } + _, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(keeper))) + return freed, err } owns, err := c.Query(ctx, "", "SELECT datname FROM pg_database WHERE datdba = (SELECT oid FROM pg_roles WHERE rolname = "+ Literal(r.Consumer)+")") diff --git a/modules/postgres/cmd/postgres-provider/retire_pg_test.go b/modules/postgres/cmd/postgres-provider/retire_pg_test.go index ee9a476a..75feec90 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg_test.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg_test.go @@ -10,31 +10,74 @@ import ( "time" ) -func TestRetiringLocksMarksAndDropsNothing(t *testing.T) { +// ADR 0570 clause 5, on the operator's yes: the login's database and objects go to its keeper, marked; its sessions +// end; the login is dropped; the database is not. +func TestRetiringHandsTheDatabaseToAKeeperAndRemovesTheLogin(t *testing.T) { f, c := newFake(admin) f.answer(`shobj_description\(oid`, []string{"c"}, []string{`{"mesh":"consumer","node":"ace"}`}) - f.answer(`SELECT 1 FROM pg_roles`, []string{"?column?"}, []string{"1"}) + f.answer(`pg_roles WHERE rolname = 'mesh_ace_letta'$`, []string{"?column?"}, []string{"1"}) + f.answer(`pg_database WHERE datname = 'mesh_ace_letta'$`, []string{"?column?"}, []string{"1"}) at := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - if err := c.RetireRole(ctx, "mesh_ace_letta", "the mesh stopped asking for it", at); err != nil { + if err := c.RetireRole(ctx, "mesh_ace_letta", "the operator approved its retirement", at); err != nil { t.Fatal(err) } - sql := f.statements() - if !has(sql, `ALTER ROLE "mesh_ace_letta" NOLOGIN`) || !has(sql, `pg_terminate_backend.*usename = 'mesh_ace_letta'`) || - !has(sql, `COMMENT ON ROLE "mesh_ace_letta" IS '\{"mesh":"consumer","node":"ace","retired":"2026-10-06T12:00:00Z","why":"the mesh stopped asking for it"\}'`) { - t.Fatal(strings.Join(sql, "\n")) + var order []string + for _, call := range f.calls { + switch { + case strings.HasPrefix(call.SQL, "SELECT"): + case strings.HasPrefix(call.SQL, "REASSIGN"), strings.HasPrefix(call.SQL, "DROP OWNED"): + order = append(order, call.Database+": "+call.SQL) + default: + order = append(order, call.SQL) + } } - noDrop(t, sql) - if has(sql, `REVOKE|ALLOW_CONNECTIONS|RENAME`) { - t.Fatal("retiring touched the database:", strings.Join(sql, "\n")) + if !has(f.statements(), `pg_terminate_backend.*usename = 'mesh_ace_letta'`) { + t.Fatal(strings.Join(f.statements(), "\n")) + } + want := []string{ + `CREATE ROLE "mesh_ace_letta~retired" NOLOGIN`, + `COMMENT ON ROLE "mesh_ace_letta~retired" IS '{"mesh":"consumer","node":"ace","retired":"2026-10-06T12:00:00Z","why":"the operator approved its retirement","login":"mesh_ace_letta"}'`, + `ALTER ROLE "mesh_ace_letta" NOLOGIN`, + `mesh_ace_letta: REASSIGN OWNED BY "mesh_ace_letta" TO "mesh_ace_letta~retired"`, + `mesh_ace_letta: DROP OWNED BY "mesh_ace_letta"`, + `DROP ROLE "mesh_ace_letta"`, + } + if strings.Join(order, "\n") != strings.Join(want, "\n") { + t.Fatalf("sent:\n%s\nwant:\n%s", strings.Join(order, "\n"), strings.Join(want, "\n")) + } + if has(f.statements(), `DROP DATABASE|REVOKE|ALLOW_CONNECTIONS|RENAME`) { + t.Fatal("retiring touched the database:", strings.Join(f.statements(), "\n")) } // No such role: nothing to retire, nothing done. f, c = newFake(admin) - if err := c.RetireRole(ctx, "gone", "x", at); err != nil || has(f.statements(), `ALTER|COMMENT`) { + if err := c.RetireRole(ctx, "gone", "x", at); err != nil || has(f.statements(), `ALTER|COMMENT|CREATE|DROP`) { t.Fatal(f.statements(), err) } } +// A login made again takes back what its keeper kept, and the keeper goes; with no keeper, nothing is sent. +func TestALoginMadeAgainReclaimsWhatItsKeeperKept(t *testing.T) { + f, c := newFake(admin) + if err := c.Reclaim(ctx, "mesh_ace_letta", "mesh_ace_letta"); err != nil || has(f.statements(), `REASSIGN|DROP`) { + t.Fatal(f.statements(), err) + } + f, c = newFake(admin) + f.answer(`pg_roles WHERE rolname = 'mesh_ace_letta~retired'$`, []string{"?column?"}, []string{"1"}) + if err := c.Reclaim(ctx, "mesh_ace_letta", "mesh_ace_letta"); err != nil { + t.Fatal(err) + } + var got []string + for _, call := range f.calls[1:] { + got = append(got, call.Database+": "+call.SQL) + } + want := `mesh_ace_letta: REASSIGN OWNED BY "mesh_ace_letta~retired" TO "mesh_ace_letta"` + "\n" + + `mesh_ace_letta: DROP OWNED BY "mesh_ace_letta~retired"` + "\n" + `postgres: DROP ROLE "mesh_ace_letta~retired"` + if strings.Join(got, "\n") != want { + t.Fatal(strings.Join(got, "\n")) + } +} + func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { f, c := newFake(admin) f.answer(`FROM pg_roles r`, []string{"rolname", "rolcanlogin", "mark", "mesh_shaped", "size"}, @@ -44,6 +87,7 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { []string{"locked_before", "f", "", "t", "400"}, []string{"hal_registry", "t", "", "f", "500"}, []string{"jochens", "t", "", "t", ""}, + []string{"mesh_ace_umami~retired", "f", `{"mesh":"consumer","node":"ace","retired":"2026-10-29T12:00:00Z","why":"approved","login":"mesh_ace_umami"}`, "f", "600"}, ) f.answer(`_deleted_`, []string{"datname", "size"}, []string{"mesh_novox_gitea_deleted_20261005", "7771827"}) inv, err := c.Inventory(ctx) @@ -53,7 +97,7 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { if strings.Join(inv.Active, ",") != "active_marked,active_shaped" { t.Fatalf("active: %v", inv.Active) } - if len(inv.Retired) != 3 { + if len(inv.Retired) != 4 { t.Fatalf("%+v", inv.Retired) } r := inv.Retired[0] @@ -64,7 +108,10 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { if r := inv.Retired[1]; r.Consumer != "locked_before" || !r.RetiredAt.IsZero() || r.Kind != harness.KindConsumer { t.Fatalf("found locked without a mark: %+v", r) } - if r := inv.Retired[2]; r.Kind != KindSetAside || r.RetiredAt.Format("20060102") != "20261005" || r.SizeBytes != 7771827 { + if r := inv.Retired[2]; r.Consumer != "mesh_ace_umami" || r.SizeBytes != 600 || r.Why != "approved" { + t.Fatalf("a keeper is listed under the login it keeps for: %+v", r) + } + if r := inv.Retired[3]; r.Kind != KindSetAside || r.RetiredAt.Format("20060102") != "20261005" || r.SizeBytes != 7771827 { t.Fatalf("set aside: %+v", r) } } @@ -97,6 +144,18 @@ func TestDeletingDropsTheDatabaseThenTheRole(t *testing.T) { if db < 0 || role < db { t.Fatal(sql) } + // A login removed on retiring: the database, then its keeper. + f, c = newFake(admin) + f.answer(`SELECT pg_database_size`, []string{"size"}, []string{"9000"}) + f.answer(`pg_roles WHERE rolname = 'mesh_ace_umami~retired'$`, []string{"?column?"}, []string{"1"}) + if _, err := c.DeleteRetired(ctx, harness.Retired{Consumer: "mesh_ace_umami", Kind: harness.KindConsumer}); err != nil { + t.Fatal(err) + } + sql = strings.Join(f.statements(), "\n") + db, keeper := strings.Index(sql, `DROP DATABASE "mesh_ace_umami"`), strings.Index(sql, `DROP ROLE "mesh_ace_umami~retired"`) + if db < 0 || keeper < db { + t.Fatal(sql) + } // A role still owning a set-aside database is kept, and said. f, c = newFake(admin) f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"f"}) diff --git a/modules/postgres/go.mod b/modules/postgres/go.mod index b1598d4e..fe566892 100644 --- a/modules/postgres/go.mod +++ b/modules/postgres/go.mod @@ -3,7 +3,7 @@ module postgres go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/jackc/pgx/v5 v5.11.0 ) diff --git a/modules/postgres/go.sum b/modules/postgres/go.sum index 923b4ba3..3d4a2c90 100644 --- a/modules/postgres/go.sum +++ b/modules/postgres/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/postgres/module.json b/modules/postgres/module.json index e8a59069..8f9f0579 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -9,6 +9,7 @@ "provides": [ { "name": "postgres-database", + "credential": {"form": "pair"}, "scope": "mesh", "identity": { "max": 63, @@ -195,6 +196,7 @@ "bound" ], "reads": [ - "mesh-controller.assignments" + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ] } diff --git a/modules/redis/module.json b/modules/redis/module.json index 58e11064..4cc6c4e5 100644 --- a/modules/redis/module.json +++ b/modules/redis/module.json @@ -7,6 +7,9 @@ "scope": "mesh", "identity": { "in": "a Redis ACL user and key prefix" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/restic/cmd/restic-backups/.gitignore b/modules/restic/cmd/restic-backups/.gitignore new file mode 100644 index 00000000..066e2897 --- /dev/null +++ b/modules/restic/cmd/restic-backups/.gitignore @@ -0,0 +1 @@ +restic-backups diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 03a4e598..f0954581 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -25,6 +25,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "os/exec" "path/filepath" @@ -32,6 +33,7 @@ import ( "slices" "strconv" "strings" + "sync" "syscall" "time" ) @@ -188,6 +190,15 @@ type Backups struct { // measuring run as processes of their own). Node is the machine's name, said in the events. Emit func(event string, body any) error Node string + // Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the + // process has no runtime to read it. + Readings func() ([]health.Reading, error) + // PutGood writes one row of the state `good` (health.GoodRow): the night's process through its own bus account, the + // tools through the runtime; nil where there is no bus to write to. + PutGood func(key string, row health.GoodRow) error + + goodMu sync.Mutex + goodWrote map[string][]byte } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index a7a3d5fc..67fb4d46 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -1,26 +1,9 @@ -// The data the modules on this machine declare, measured (novox/hq ADR 0233). +// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them. // -// The mesh composes a second file beside the backup lines: every data item every module on the -// machine declares, one line each, -// -// item -// -// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump -// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and -// measure is how the item is measured. This holder measures each item that is not a cache — its size, -// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) — -// and says it, with each module's last good backup and the precision of the measurement, in -// `backed-up`. -// -// **Never an unbounded walk of something large.** Three methods, the item's own choice: -// -// - `walk` (the default, for small items): every file summed and the newest change found, in one walk -// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially"; -// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and -// the newest change among the path's top-level entries; hourly, and nothing is walked; -// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written, -// -// goes without a backup, or is replaced by an empty copy of itself; this holder only measures. +// The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line +// each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its +// seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's +// last good backup, in `backed-up`. // // And it deletes, when a person has decided: an item the mesh retired — its module gone from this // machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been @@ -32,104 +15,32 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "regexp" - "strconv" "strings" "sync" "time" ) -// Item is one data item a module declares. -type Item struct { - // Measure is walk, dataset or shallow. - Measure string `json:"measure,omitempty"` - Module string `json:"-"` - ID string `json:"item"` - Class string `json:"class"` - Path string `json:"path"` - CoveredBy string `json:"covered_by,omitempty"` - Protection string `json:"protection,omitempty"` -} +// Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text). +type Item = catalogue.PlacedDataItem -// Redundancy is the redundant storage an item is on, as read here. -type Redundancy struct { - // Kind is zfs, md or btrfs. - Kind string `json:"kind"` - // Where is the pool, the array device or the filesystem. - Where string `json:"where"` - // Healthy is nil when its state could not be read. - Healthy *bool `json:"healthy"` - Said string `json:"said"` -} - -// Measured is what one measurement found. -type Measured struct { - SizeBytes *int64 `json:"size_bytes,omitempty"` - LastWrite *time.Time `json:"last_write,omitempty"` - MeasuredAt *time.Time `json:"measured_at,omitempty"` - // Precision says what the size is: "exact", "dataset : its whole size", "partial: …" (a lower - // bound, never compared), or "no size: …". - Precision string `json:"precision,omitempty"` - Error string `json:"error,omitempty"` - Redundancy *Redundancy `json:"redundancy,omitempty"` -} - -// ItemReport is one item as `backed-up` says it. +// ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and +// its newest good backup. type ItemReport struct { Item - Measured + health.Measured + // Peak is the largest size in the last seven days, on PeakDay. + Peak *int64 `json:"peak_bytes,omitempty"` + PeakDay string `json:"peak_day,omitempty"` LastBackup *time.Time `json:"last_backup,omitempty"` } -// The classes the mesh declares; a cache is listed and never measured. -const classCache = "cache" - var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`) -// parseItems reads the composed data file into each module's items. A line this holder does not read -// is refused, naming the module, as a backup line is. -func parseItems(text string) (map[string][]Item, error) { - out := map[string][]Item{} - module := "" - for _, raw := range strings.Split(text, "\n") { - line := strings.TrimSpace(raw) - if line == "" { - continue - } - if m := moduleHeader.FindStringSubmatch(line); m != nil { - module = m[1] - continue - } - if strings.HasPrefix(line, "#") || module == "" { - continue - } - f := strings.Fields(line) - if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { - return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) - } - it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]} - if len(f) >= 6 { - it.Protection = f[5] - } - it.Measure = measureWalk - if len(f) == 7 { - it.Measure = f[6] - } - switch it.Measure { - case measureWalk, measureDataset, measureShallow: - default: - return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) - } - if f[4] != "-" { - it.CoveredBy = f[4] - } - out[module] = append(out[module], it) - } - return out, nil -} - // Items is what the modules on this machine declare; none when the mesh composed no data file — an // older controller, which composes none. func (b *Backups) Items() (map[string][]Item, error) { @@ -143,257 +54,30 @@ func (b *Backups) Items() (map[string][]Item, error) { if err != nil { return nil, err } - return parseItems(string(raw)) + return catalogue.ParseDataItems(string(raw)) } -func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") } - -// Measurements is the newest measurement of each item, by module and item. -func (b *Backups) Measurements() map[string]map[string]Measured { - out := map[string]map[string]Measured{} - if raw, err := os.ReadFile(b.measuredFile()); err == nil { - _ = json.Unmarshal(raw, &out) +// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state +// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading. +func (b *Backups) Measurements() map[string]map[string]health.Reading { + out := map[string]map[string]health.Reading{} + if b.Readings == nil { + return out + } + rs, err := b.Readings() + if err != nil { + b.Say("disk-load's readings could not be read: %v", err) + return out + } + for _, r := range rs { + if out[r.Module] == nil { + out[r.Module] = map[string]health.Reading{} + } + out[r.Module][r.ID] = r } return out } -// The ways an item is measured. -const ( - measureWalk = "walk" - measureDataset = "dataset" - measureShallow = "shallow" -) - -// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large -// one stops and says so rather than loading the disks for hours. -var ( - walkFor = 10 * time.Minute - walkFiles = 2_000_000 - walkEvery = 23 * time.Hour -) - -// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded -// by time and by files. One line out: " ". -func walkCommand(path string) string { - return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ - "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ - "echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles) -} - -// topCommand is the newest change among a path and its top-level entries, and how many there are: -// one directory read, bounded. " ". -func topCommand(path string) string { - return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + - "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" -} - -func epochTime(s string) *time.Time { - epoch, err := strconv.ParseInt(s, 10, 64) - if err != nil || epoch <= 0 { - return nil - } - t := time.Unix(epoch, 0).UTC() - return &t -} - -// measure is one item, measured now, by its own method. -func (b *Backups) measure(ctx context.Context, it Item) Measured { - at := b.Now().UTC() - m := Measured{MeasuredAt: &at} - if !b.exists(ctx, it.Path) { - m.Error = it.Path + " does not exist" - zero := int64(0) - m.SizeBytes, m.Precision = &zero, "exact" - return m - } - m.Redundancy = b.redundancyOf(ctx, it.Path) - switch it.Measure { - case measureDataset, measureShallow: - out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) - if f := strings.Fields(out); err == nil && len(f) == 2 { - m.LastWrite = epochTime(f[0]) - } else if err != nil { - m.Error = err.Error() - } - m.Precision = "no size: the newest change among its top-level entries only" - if it.Measure == measureShallow { - return m - } - out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) - f := strings.Fields(out) - if err != nil || len(f) != 2 { - m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" - return m - } - used, err := strconv.ParseInt(f[1], 10, 64) - if err != nil { - return m - } - m.SizeBytes = &used - m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" - return m - } - out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) - if err != nil { - m.Error = err.Error() - return m - } - lines := strings.Split(strings.TrimSpace(out), "\n") - f := strings.Fields(lines[0]) - if len(f) != 3 || len(lines) < 2 { - m.Error = "the measurement said " + strings.TrimSpace(out) - return m - } - size, err1 := strconv.ParseInt(f[0], 10, 64) - files, err2 := strconv.Atoi(f[2]) - if err1 != nil || err2 != nil { - m.Error = "the measurement said " + strings.TrimSpace(out) - return m - } - m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) - m.Precision = "exact" - if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles { - m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ - "and this item wants measure: dataset or shallow", walkFor, walkFiles) - } - return m -} - -// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own -// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no -// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the -// controller says is no protection at all. -func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy { - out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) - if err != nil { - return nil - } - f := strings.Fields(out) - if len(f) < 2 { - return nil - } - source, fstype := f[0], f[1] - yes, no := true, false - switch { - case fstype == "zfs": - pool, _, _ := strings.Cut(source, "/") - r := &Redundancy{Kind: "zfs", Where: pool} - health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) - if err != nil { - r.Said = "zpool list: " + err.Error() - return r - } - status, err := b.Run(ctx, "zpool", "status", "-x", pool) - if err != nil { - r.Said = "zpool status: " + err.Error() - return r - } - health, status = strings.TrimSpace(health), strings.TrimSpace(status) - r.Said = "health " + health + "; " + firstLineOf(status) - if health == "ONLINE" && strings.Contains(status, "is healthy") { - r.Healthy = &yes - } else { - r.Healthy = &no - r.Said = "health " + health + "; " + oneLineOf(status) - } - return r - case strings.HasPrefix(source, "/dev/md"): - device := strings.TrimPrefix(source, "/dev/") - r := &Redundancy{Kind: "md", Where: device} - mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") - if err != nil { - r.Said = err.Error() - return r - } - healthy, said, found := mdHealth(mdstat, device) - if !found { - r.Said = device + " is not in /proc/mdstat" - return r - } - r.Said = said - if healthy { - r.Healthy = &yes - } else { - r.Healthy = &no - } - return r - case fstype == "btrfs": - r := &Redundancy{Kind: "btrfs", Where: source} - stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) - if err != nil { - r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) - return r - } - r.Healthy, r.Said = &yes, firstLineOf(stats) - return r - } - return nil -} - -var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) - -// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not -// recovering. -func mdHealth(mdstat, device string) (bool, string, bool) { - lines := strings.Split(mdstat, "\n") - for i, l := range lines { - if !strings.HasPrefix(l, device+" ") { - continue - } - block := l - for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { - block += " " + strings.TrimSpace(lines[j]) - } - members := mdMembers.FindStringSubmatch(block) - healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") - return healthy, oneLineOf(block), true - } - return false, "", false -} - -func firstLineOf(s string) string { - line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") - return line -} - -func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } - -// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item -// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the -// hourly round reads counters, and a walk runs at most once a day. -func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { - // The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454). - release, err := b.hold(measureLock) - if err != nil { - return err - } - defer release() - items, err := b.Items() - if err != nil { - return err - } - before := b.Measurements() - found := map[string]map[string]Measured{} - for module, its := range items { - for _, it := range its { - if it.Class == classCache { - continue - } - if found[module] == nil { - found[module] = map[string]Measured{} - } - if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil && - b.Now().Sub(*was.MeasuredAt) < walkEvery { - found[module][it.ID] = was - continue - } - found[module][it.ID] = b.measure(ctx, it) - } - } - raw, _ := json.MarshalIndent(found, "", " ") - return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600) -} - func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") } // Good is each module's newest good night: what a night that failed since does not erase. @@ -418,14 +102,18 @@ func (b *Backups) recordGood(module string, at time.Time) { if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording %s's good night failed: %v", module, err) } + b.keepGood() } -// itemReports is every item of one module, with its newest measurement and its newest good backup: the +// itemReports is every item of one module, with its newest reading and its newest good backup: the // module's newest good night, where that night keeps the path that covers the item. -func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport { +func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport { out := []ItemReport{} for _, it := range its { - r := ItemReport{Item: it, Measured: measured[it.ID]} + r := ItemReport{Item: it} + if m, ok := measured[it.ID]; ok { + r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt + } if it.CoveredBy != "" && !good.IsZero() { for _, p := range paths { if p == it.CoveredBy { diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index 0834ad61..9e41411d 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -6,7 +6,8 @@ package main import ( "context" "errors" - "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "strings" @@ -28,53 +29,34 @@ func withData(t *testing.T, declared, data string) Where { } func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) { - got, err := parseItems(composedData) + b := &Backups{Where: withData(t, composed, composedData), Now: time.Now, Say: quiet} + got, err := b.Items() must(t, err) if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" || got["searxng"][0].Class != "cache" { t.Fatalf("%+v", got) } - for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} { - if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") { - t.Errorf("%q: %v", bad, err) - } - } - // An older controller composes no data file: nothing is measured, nothing fails. - b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} + // An older controller composes no data file: nothing is declared, nothing fails. + b = &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} if items, err := b.Items(); err != nil || len(items) != 0 { t.Fatalf("%v, %v", items, err) } } -// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and -// `backed-up` says each with the newest good night of the module that keeps the path covering it. -func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) { +// `backed-up` says every item with disk-load's newest reading and its peak, and the newest good night of the module that +// keeps the path covering it; an item with no reading yet is said with none. +func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) { where := withData(t, composed, composedData) - var mu sync.Mutex - var walked []string - run := func(_ context.Context, name string, args ...string) (string, error) { - mu.Lock() - defer mu.Unlock() - switch { - case name == "test": - return "", nil - case name == "bash" && strings.Contains(args[1], "find '"): - walked = append(walked, args[1]) - if strings.Contains(args[1], "'/var/lib/mesh-store'") { - return "1073741824 1759744800 4000\n0\n", nil - } - return "5000 1759700000 3\n0\n", nil - case name == "restic": - return "[]", nil - } - return "", nil - } + size, peak := int64(1<<30), int64(2<<30) + at := time.Date(2026, 10, 6, 4, 0, 0, 0, time.UTC) + write := time.Unix(1759744800, 0).UTC() night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) - b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} - must(t, b.MeasureAll(context.Background(), true)) - if len(walked) != 2 { - t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) - } + b := &Backups{Where: where, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet, + Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }, + Readings: func() ([]health.Reading, error) { + return []health.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"}, + Measured: health.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil + }} b.recordGood("postgres", night) got, err := b.BackedUp(context.Background(), "") must(t, err) @@ -89,12 +71,23 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) } } } - if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil || - store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { + if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.Peak == nil || *store.Peak != 2<<30 || + store.LastWrite == nil || store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { t.Fatalf("the store is said as %+v", store) } - if dumps == nil || dumps.LastBackup != nil { - t.Fatalf("an item not backed up is said backed up: %+v", dumps) + if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil { + t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps) + } + // The same last good night is kept on the bus beside good.json, one row per item a backup path covers. + b.Node = "ace" + wrote := map[string]health.GoodRow{} + b.PutGood = func(k string, r health.GoodRow) error { wrote[k] = r; return nil } + b.recordGood("postgres", night) // a good night writes the rows, once; the same night again writes nothing + n := len(wrote) + b.recordGood("postgres", night) + row, ok := wrote[health.GoodKey("ace", "postgres", "store")] + if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != health.GoodVersion { + t.Fatalf("the good rows are %+v", wrote) } } @@ -202,104 +195,3 @@ func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) { t.Fatalf("%+v, %v", again, err) } } - -// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a -// member missing; and plain storage, which is no redundancy at all. -func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { - health, statusX := "ONLINE\n", "pool 'storage' is healthy\n" - fs := "storage/media zfs\n" - run := func(_ context.Context, name string, args ...string) (string, error) { - switch name { - case "findmnt": - return fs, nil - case "zpool": - if args[0] == "list" { - return health, nil - } - return statusX, nil - case "cat": - return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil - } - return "", nil - } - b := &Backups{Run: run, Now: time.Now, Say: quiet} - r := b.redundancyOf(context.Background(), "/storage/media/movies") - if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy { - t.Fatalf("a healthy pool: %+v", r) - } - health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" - if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { - t.Fatalf("a degraded pool: %+v", r) - } - fs = "/dev/md127 ext4\n" - if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { - t.Fatalf("an array missing a member: %+v", r) - } - fs = "/dev/nvme0n1p3 ext4\n" - if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil { - t.Fatalf("plain storage read as redundant: %+v", r) - } -} - -// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level -// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops -// and says so; and a walk runs at most once a day — the hourly round keeps the last one. -func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) { - data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" + - "item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" + - "item big valuable /srv/big /srv/big backup\n" - where := withData(t, composed, data) - var mu sync.Mutex - var walks []string - truncated := true - run := func(_ context.Context, name string, args ...string) (string, error) { - mu.Lock() - defer mu.Unlock() - line := name + " " + strings.Join(args, " ") - switch { - case name == "test": - return "", nil - case name == "zfs" && args[len(args)-1] == "/storage/media/movies": - return "storage/media\t97067690722560\n", nil - case name == "zfs": - return "", errors.New("not a ZFS dataset") - case name == "bash" && strings.Contains(line, "-maxdepth 1"): - return "1759744800 12673\n", nil - case name == "bash": - walks = append(walks, line) - if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") { - t.Errorf("walked something declared not to be walked: %s", line) - } - if truncated { - return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil - } - return "999 1759700000 10\n0\n", nil - } - return "", nil - } - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet} - must(t, b.MeasureAll(context.Background(), false)) - got := b.Measurements()["media"] - if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil { - t.Fatalf("the library from its dataset: %+v", m) - } - if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil { - t.Fatalf("a shallow item: %+v", m) - } - if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") { - t.Fatalf("a walk stopped at its bound: %+v", m) - } - // An hour later the hourly round reads counters again and walks nothing. - now = now.Add(time.Hour) - truncated = false - must(t, b.MeasureAll(context.Background(), false)) - if len(walks) != 1 { - t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks) - } - // After the night, it walks. - must(t, b.MeasureAll(context.Background(), true)) - if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" { - t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"]) - } -} diff --git a/modules/restic/cmd/restic-backups/good.go b/modules/restic/cmd/restic-backups/good.go new file mode 100644 index 00000000..5ea3c180 --- /dev/null +++ b/modules/restic/cmd/restic-backups/good.go @@ -0,0 +1,128 @@ +package main + +// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's +// own state `good` (health.GoodRow), one key `..`, written when a night's files change, so the +// self-check's probe of a stale backup reads it there. + +import ( + "encoding/json" + "path/filepath" + "strings" + "syscall" + "time" + "unsafe" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// GoodRows is the rows of this machine (health.GoodRow): every item a module declares that a path of the module's backup +// covers, with the module's newest good night. +func (b *Backups) GoodRows() (map[string]health.GoodRow, error) { + items, err := b.Items() + if err != nil { + return nil, err + } + declared, err := b.Declared() + if err != nil { + return nil, err + } + paths := map[string][]string{} + for _, d := range declared { + paths[d.Module] = d.Paths + } + good := b.Good() + out := map[string]health.GoodRow{} + for module, its := range items { + for _, it := range its { + if it.Class == "cache" || it.CoveredBy == "" { + continue + } + covered := false + for _, p := range paths[module] { + covered = covered || p == it.CoveredBy + } + if !covered { + continue + } + out[health.GoodKey(b.Node, module, it.ID)] = health.GoodRow{V: health.GoodVersion, Node: b.Node, Module: module, + Item: it.ID, CoveredBy: it.CoveredBy, LastGood: good[module].UTC()} + } + } + return out, nil +} + +// keepGood writes the rows that differ from what was last written, when a night succeeded (recordGood). A bus that +// cannot be reached loses nothing: good.json is the local copy, and the next good night writes the rows again. +func (b *Backups) keepGood() { + if b.PutGood == nil { + return + } + rows, err := b.GoodRows() + if err != nil { + b.Say("the last good nights could not be read: %v", err) + return + } + b.goodMu.Lock() + defer b.goodMu.Unlock() + if b.goodWrote == nil { + b.goodWrote = map[string][]byte{} + } + for k, r := range rows { + raw, _ := json.Marshal(r) + if string(b.goodWrote[k]) == string(raw) { + continue + } + if err := b.PutGood(k, r); err != nil { + b.Say("the last good night of %s was not put on the bus: %v", k, err) + continue + } + b.goodWrote[k] = raw + } +} + +// watchGood writes the rows once at start and then each time the night's process changes what it keeps in the state +// directory (good.json, the nights' file), as the file system says it: the night runs as the operator account, which a +// process with a bus account of its own may not (ADR 0436 §2), so the tools' runtime, which has the module's account, +// writes at the night's event instead of the night itself. No timer. +func watchGood(b *Backups) { + time.Sleep(20 * time.Second) // the runtime serves first + b.keepGood() + for { + err := whenWritten(b.Where.State, func(name string) bool { return name == "good.json" || name == filepath.Base(b.nightsFile()) }, b.keepGood) + b.Say("watching the state directory for a night failed (%v); trying again in a minute", err) + time.Sleep(time.Minute) + b.keepGood() + } +} + +// whenWritten calls then each time a file of dir that match names is written or put in place, until the watch fails. +func whenWritten(dir string, match func(name string) bool, then func()) error { + fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC) + if err != nil { + return err + } + defer syscall.Close(fd) + if _, err := syscall.InotifyAddWatch(fd, dir, syscall.IN_CLOSE_WRITE|syscall.IN_MOVED_TO); err != nil { + return err + } + buf := make([]byte, 64*syscall.SizeofInotifyEvent+4096) + for { + n, err := syscall.Read(fd, buf) + if err != nil { + if err == syscall.EINTR { + continue + } + return err + } + heard := false + for off := 0; off+syscall.SizeofInotifyEvent <= n; { + ev := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[off])) + name := strings.TrimRight(string(buf[off+syscall.SizeofInotifyEvent:off+syscall.SizeofInotifyEvent+int(ev.Len)]), "\x00") + heard = heard || match(name) + off += syscall.SizeofInotifyEvent + int(ev.Len) + } + if heard { + then() + } + } +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 838aa88e..76ce1550 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -263,8 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { } env := m.Build.Artifacts[0].Env want := map[string]struct{ run, kind, tool string }{ - "night": {"night", "tool", "backup_health"}, - "measure": {"measure", "unit", ""}, + "night": {"night", "unit", ""}, + "awake": {"awake", "tool", "backup_health"}, } for _, r := range m.Resources { w, ok := want[r["id"].(string)] diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index f4e08e32..20c0a76b 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -24,6 +24,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/health" ) // Seat is the role this module holds. @@ -46,20 +48,29 @@ func main() { } if way == "" { b.Emit = func(event string, body any) error { return stdio.Emit(event, body) } + b.Readings = busReadings(b.Node) } switch way { case "night": nights(b) - case "measure": - measurements(b) + case "awake": + counting(b) case "": + if err := stdio.Keeps(announce.State{Name: health.GoodState}); err != nil { + say("%v", err) + os.Exit(1) + } + b.PutGood = func(key string, row health.GoodRow) error { + _, err := stdio.State(health.GoodState).Put(key, row) + return err + } + go watchGood(b) if err := stdio.Serve("", tools(b)); err != nil { say("%v", err) os.Exit(1) } default: - say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+ - "measure it measures the data declared here", way) + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with awake it counts how long the machine is up since a night", way) os.Exit(2) } } @@ -72,7 +83,6 @@ func nights(b *Backups) { if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { hour = h } - go counting(b) if missedANight(b.Nights(), time.Now()) { time.Sleep(10 * time.Minute) night(b) @@ -83,23 +93,6 @@ func nights(b *Backups) { } } -// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it), -// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after -// each night (novox/hq ADR 0233). -func measurements(b *Backups) { - every := time.Hour - if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute { - every = d - } - time.Sleep(3 * time.Minute) - for { - if err := b.MeasureAll(context.Background(), false); err != nil { - say("measuring the data declared here failed: %v", err) - } - time.Sleep(every) - } -} - func night(b *Backups) { ctx := context.Background() outcome, err := b.BackUp(ctx, "") @@ -116,9 +109,6 @@ func night(b *Backups) { if len(failed) > 0 { say("the night left %s without a backup", strings.Join(failed, ", ")) } - if err := b.MeasureAll(ctx, true); err != nil { - say("measuring the data declared here failed: %v", err) - } // Sundays, the repository's own integrity with a sample of the data read back. if time.Now().Weekday() == time.Sunday { if err := b.Check(ctx); err != nil { @@ -204,3 +194,28 @@ func tools(b *Backups) []stdio.Tool { Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }}, } } + +// busReadings reads disk-load's readings of this machine from its state, every key `..`. +func busReadings(node string) func() ([]health.Reading, error) { + return func() ([]health.Reading, error) { + st := stdio.State(health.ReadingsRead) + keys, err := st.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := st.Get(k) + if err != nil || e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil + } +} diff --git a/modules/restic/go.mod b/modules/restic/go.mod index 7ec16d84..cc18cada 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -1,5 +1,5 @@ module restic -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/restic/go.sum b/modules/restic/go.sum index dfc73823..f8bfbd2b 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/restic/module.json b/modules/restic/module.json index ea5ec8b9..05927fe2 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -41,40 +41,40 @@ } ] }, - "config": { - "keep-daily": { - "type": "number", - "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 14 - }, - "keep-weekly": { - "type": "number", - "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 8 - }, - "keep-monthly": { - "type": "number", - "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 6 - }, - "keep-hand-taken-days": { - "type": "number", - "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 7 - } + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } + }, "resources": [ { "id": "state", @@ -139,22 +139,17 @@ "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, "health": { - "kind": "tool", - "tool": "backup_health", - "interval": "60s", - "timeout": "20s", - "looks": 2, - "grace": "60s" + "kind": "unit" } }, { - "id": "measure", + "id": "awake", "type": "process", - "name": "restic-measure", + "name": "restic-awake", "artifact": "tools", "run": [ "./restic-backups", - "measure" + "awake" ], "user": "${machine:account}", "env": { @@ -165,7 +160,12 @@ "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, "health": { - "kind": "unit" + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" } } ], @@ -195,5 +195,11 @@ "from": "files" } ] - } + }, + "reads": [ + "disk-load.readings" + ], + "state": [ + "good" + ] } diff --git a/modules/route-adapter/cmd/route-provider/provider_test.go b/modules/route-adapter/cmd/route-provider/provider_test.go index abb464e1..44ac3e40 100644 --- a/modules/route-adapter/cmd/route-provider/provider_test.go +++ b/modules/route-adapter/cmd/route-provider/provider_test.go @@ -7,6 +7,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // bucketBus is the helper's bus over in-memory buckets. @@ -45,12 +46,22 @@ func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { // the entry's values are the controller's composition, copied unchanged, and no vault is asked. func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { bus := &bucketBus{buckets: map[string]map[string][]byte{}} - plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{ + plan, _ := json.Marshal(map[string]any{"v": 1, "node": "g14", "module": "gitea", "plan": map[string]any{ "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea", "names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal", "name-web": "git.example.org"}}}}}) - if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil { + if err := bus.Put(context.Background(), provisions.PlansBucket, "g14.gitea", plan); err != nil { + t.Fatal(err) + } + // The key the seat composed for the entry, asked of route-adapter (novox/hq ADR 0570 clauses 1, 3). + key, _ := provisions.Key("g14", "gitea", ProvisionName, "") + asked, err := provisions.Encode(provisions.Provision{Provision: ProvisionName, State: provisions.Asked, + Consumer: provisions.End{Node: "g14", Module: "gitea"}, Provider: provisions.End{Node: "novox", Module: ModuleName}}) + if err != nil { + t.Fatal(err) + } + if err := bus.Put(context.Background(), provisions.ProvisionsBucket, key, asked); err != nil { t.Fatal(err) } p := newProvider("novox", bus, t.Logf) diff --git a/modules/route-adapter/go.mod b/modules/route-adapter/go.mod index a4ec98b8..a4fe90d2 100644 --- a/modules/route-adapter/go.mod +++ b/modules/route-adapter/go.mod @@ -2,4 +2,4 @@ module route-adapter go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/route-adapter/go.sum b/modules/route-adapter/go.sum index 6162079d..f8bfbd2b 100644 --- a/modules/route-adapter/go.sum +++ b/modules/route-adapter/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/route-adapter/module.json b/modules/route-adapter/module.json index 5234c78b..2827462c 100644 --- a/modules/route-adapter/module.json +++ b/modules/route-adapter/module.json @@ -9,6 +9,7 @@ "provides": [ { "name": "route", + "credential": {"form": "none"}, "scope": "mesh", "identity": false } @@ -129,6 +130,7 @@ "bound" ], "reads": [ - "mesh-controller.assignments" + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ] } diff --git a/modules/showcase/module.json b/modules/showcase/module.json index e657e2ab..6b0db672 100644 --- a/modules/showcase/module.json +++ b/modules/showcase/module.json @@ -9,7 +9,10 @@ "provides": [ { "name": "greeting", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "serves": { diff --git a/modules/step-ca/module.json b/modules/step-ca/module.json index b11d2b99..3f44ead7 100644 --- a/modules/step-ca/module.json +++ b/modules/step-ca/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "internal-acme-ca", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "serves": { diff --git a/modules/systemd-resolved/go.mod b/modules/systemd-resolved/go.mod index c50c4f22..79ddf566 100644 --- a/modules/systemd-resolved/go.mod +++ b/modules/systemd-resolved/go.mod @@ -1,5 +1,5 @@ module systemd-resolved -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/systemd-resolved/go.sum b/modules/systemd-resolved/go.sum index b474419e..f8bfbd2b 100644 --- a/modules/systemd-resolved/go.sum +++ b/modules/systemd-resolved/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/systemd-resolved/module.json b/modules/systemd-resolved/module.json index b850afcf..24b3bc85 100644 --- a/modules/systemd-resolved/module.json +++ b/modules/systemd-resolved/module.json @@ -8,7 +8,10 @@ "provides": [ { "name": "split-dns", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "requires": [ diff --git a/modules/umami/module.json b/modules/umami/module.json index 22ead419..e3b9aa0d 100644 --- a/modules/umami/module.json +++ b/modules/umami/module.json @@ -32,7 +32,10 @@ "provides": [ { "name": "analytics", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "serves": { diff --git a/modules/xorg/go.mod b/modules/xorg/go.mod index 356d1874..95dbdc00 100644 --- a/modules/xorg/go.mod +++ b/modules/xorg/go.mod @@ -1,5 +1,5 @@ module xorg -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/xorg/go.sum b/modules/xorg/go.sum index b474419e..f8bfbd2b 100644 --- a/modules/xorg/go.sum +++ b/modules/xorg/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/xorg/module.json b/modules/xorg/module.json index 7b0b6b1e..1685dcb4 100644 --- a/modules/xorg/module.json +++ b/modules/xorg/module.json @@ -9,7 +9,10 @@ "provides": [ { "name": "x11-display", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "claims": [