diff --git a/checks/handacts/fallbacks b/checks/handacts/fallbacks new file mode 100644 index 00000000..c4fdf709 --- /dev/null +++ b/checks/handacts/fallbacks @@ -0,0 +1,8 @@ +# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each ` `: +# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until +# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any +# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write +# (novox/hq ADR 0565 §11; issue 1953, issue 774). +mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first +mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first +mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first diff --git a/checks/handacts/go.mod b/checks/handacts/go.mod new file mode 100644 index 00000000..29e4c52b --- /dev/null +++ b/checks/handacts/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/handacts + +go 1.22 diff --git a/checks/handacts/main.go b/checks/handacts/main.go new file mode 100644 index 00000000..1a222d60 --- /dev/null +++ b/checks/handacts/main.go @@ -0,0 +1,109 @@ +// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every +// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes +// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the +// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and +// an entry whose module no longer invokes the verb is refused too. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks. +var Verbs = []string{"hand-act", "warranted", "drill"} + +const ( + invokePrefix = "seat:mesh-controller." + Allowed = "checks/handacts/fallbacks" +) + +// Findings is every refusal in the catalog at root, sorted. +func Findings(root string) ([]string, error) { + allowed := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(root, Allowed)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]+" "+f[1]] = true + } else if len(f) > 0 && !strings.HasPrefix(f[0], "#") { + return nil, fmt.Errorf("%s: %q is not ` `", Allowed, line) + } + } + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var out []string + invoked := map[string]bool{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Invokes []string `json:"invokes"` + } + if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" { + continue + } + for _, inv := range m.Invokes { + verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix) + if !ok || !contains(Verbs, verb) { + continue + } + invoked[m.Module+" "+verb] = true + if !allowed[m.Module+" "+verb] { + out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+ + "or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed)) + } + } + } + for entry := range allowed { + if !invoked[entry] { + out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks", + Allowed, entry)) + } + } + sort.Strings(out) + return out, nil +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: handacts ") + os.Exit(2) + } + found, err := Findings(os.Args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } + fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named") +} diff --git a/checks/handacts/main_test.go b/checks/handacts/main_test.go new file mode 100644 index 00000000..61e197e6 --- /dev/null +++ b/checks/handacts/main_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is +// refused (novox/hq ADR 0565 §11's catalog test). +func TestFindings(t *testing.T) { + for _, row := range []struct { + name string + files map[string]string + want string // empty: clean + }{ + {"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""}, + {"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`}, + "a invokes seat:mesh-controller.warranted"}, + {"a named fallback, and the controller's own", map[string]string{ + "modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`, + "modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`, + Allowed: "# why\na drill fallback until the lease write\n"}, ""}, + {"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`}, + {"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""}, + } { + t.Run(row.name, func(t *testing.T) { + root := t.TempDir() + for path, text := range row.files { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := Findings(root) + if err != nil { + t.Fatal(err) + } + if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) { + t.Fatalf("findings = %q, want one containing %q", all, row.want) + } + }) + } +} + +// The run of this check over the real tree. +func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) { + got, err := Findings(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + for _, f := range got { + t.Error(f) + } +} diff --git a/checks/handacts/reads-outside b/checks/handacts/reads-outside new file mode 100644 index 00000000..61f9814a --- /dev/null +++ b/checks/handacts/reads-outside @@ -0,0 +1,3 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what it invokes. +modules/*/module.json diff --git a/checks/health/go.mod b/checks/health/go.mod index 4507ee45..8c3f521e 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 diff --git a/checks/health/go.sum b/checks/health/go.sum index 3eadb468..387b26bd 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 h1:+jH77IJmxUiFDdbJljhynQRY0Cc5/5jtwmMV7UUqRI4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-conditions/cmd/mesh-conditions/compare.go b/modules/mesh-conditions/cmd/mesh-conditions/compare.go new file mode 100644 index 00000000..f2092381 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/compare.go @@ -0,0 +1,218 @@ +package main + +// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare` +// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the +// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and +// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a +// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the +// controller until the lease write, and is refused after it. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Difference is one key the two sides do not agree on, or one listed apart. +type Difference struct { + Key string `json:"key"` + Kind string `json:"kind"` + Reason string `json:"reason"` +} + +// Comparison is compare's answer. +type Comparison struct { + Differences []Difference `json:"differences"` + // Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held. + Apart []Difference `json:"apart"` + Same int `json:"same"` + At time.Time `json:"at"` +} + +// Empty says nothing differs. +func (c Comparison) Empty() bool { return len(c.Differences) == 0 } + +// Compare is the keeper's open conditions against the controller's. +func (k *Keeper) Compare() (Comparison, error) { + mine, err := k.Open() + if err != nil { + return Comparison{}, err + } + k.mu.Lock() + k.init() + theirs := map[string]health.Condition{} + for key, c := range k.ctrl { + theirs[key] = c + } + heldNow := map[string]held{} + for key, h := range k.held { + heldNow[key] = h + } + k.mu.Unlock() + out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()} + open := map[string]health.Condition{} + for _, c := range mine { + open[c.Key] = c + } + keys := map[string]bool{} + for key := range open { + keys[key] = true + } + for key := range theirs { + keys[key] = true + } + for key := range heldNow { + keys[key] = true + } + sorted := make([]string, 0, len(keys)) + for key := range keys { + sorted = append(sorted, key) + } + sort.Strings(sorted) + for _, key := range sorted { + c, inMine := open[key] + t, inTheirs := theirs[key] + kind := c.Kind + if !inMine { + kind = t.Kind + } + home, homed := k.homeOf(kind) + if h, ok := heldNow[key]; ok { + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))}) + continue + } + switch { + case homed && (home.Carried || home.Replaced): + why := "ends with the controller: carried from its bucket until the lease write" + if home.Replaced { + why = "ends with the controller: its fault is observed otherwise" + } + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why}) + case inMine && !inTheirs && (!homed || !home.Controller): + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"}) + case inMine && !inTheirs: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: "open at the keeper, not at the controller"}) + case !inMine && !homed: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)}) + case !inMine: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)}) + case c.Severity != t.Severity: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)}) + default: + out.Same++ + } + } + return out, nil +} + +// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands. +type Asks interface { + Ask(a asks.Ask) error +} + +// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write. +func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a take-over says why") + } + if k.LeaseAway() { + return nil, errors.New("the controller's lease is handed away: the keeper is the one source already") + } + cmp, err := k.Compare() + if err != nil { + return nil, err + } + if !cmp.Empty() { + return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none", + len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason) + } + now := k.Now().UTC() + id := fmt.Sprintf("take-over-%d", now.UnixNano()) + act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id} + bind, err := asks.ActDigest(act) + if err != nil { + return nil, err + } + ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator, + Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+ + "the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+ + "gives up its lease.", cmp.Same), + Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}}, + Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.", + About: Module + "-take-over", Details: "Asked by " + by + ": " + why} + if err := a.Ask(ask); err != nil { + return nil, err + } + return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil +} + +// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper. +func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error { + if o.ID != "yes" { + k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID) + return nil + } + cmp, err := k.Compare() + if err != nil { + return err + } + if !cmp.Empty() { + return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask, + len(cmp.Differences)) + } + return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask}) +} + +// HandBack switches the readers back to the controller, until the lease write. +func (k *Keeper) HandBack(by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a hand-back says why") + } + if k.LeaseAway() { + return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to") + } + t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why} + if err := k.writeTakeover(t); err != nil { + return nil, err + } + return map[string]any{"follow": t.Follow, "at": t.At}, nil +} + +// writeTakeover writes the take-over key, and keeps it in history as the record of the switch. +func (k *Keeper) writeTakeover(t health.Takeover) error { + raw, err := health.EncodeTakeover(t) + if err != nil { + return err + } + _, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return err + } + if found { + _, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev) + } else { + _, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw)) + } + if errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("the take-over key moved under this write: read it again and ask again") + } + if err != nil { + return err + } + k.Takeover(t) + k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By) + return nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go new file mode 100644 index 00000000..8491812e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -0,0 +1,340 @@ +package main + +// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12; +// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a +// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or +// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state +// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once, +// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's +// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery +// did not end. A finding about another module or about the machine itself is never held. A condition already open is +// not held: it is no news. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The modules whose delivery holds what they serve beyond their own findings (Limits 12). +const ( + controllerModule = "mesh-controller" + runtimeModule = "node-tools" +) + +// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its +// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names. +var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true} + +// about is a node and a module a finding or a condition is about, for the hold. +type about struct{ Node, Module string } + +// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its +// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is +// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself. +func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about { + var out []about + parts := strings.Split(s.ID, ".") + switch s.Scope { + case health.ScopeModule: + if len(parts) >= 2 { + out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) + } + case health.ScopeProvider: + if len(parts) >= 2 { + out = append(out, about{Node: parts[1], Module: parts[0]}) + } + case health.ScopeSeat: + // The module the placement says holds the seat on that node, or the seat's own name where none is read. + if len(parts) >= 2 { + node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".") + module := k.holders[node+"."+seat] + if module == "" { + module = seat + } + out = append(out, about{Node: node, Module: module}) + } + case health.ScopeMachine: + // The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is + // delivered. + if source == "S11" || kind == "tools-silent" { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + if s.Machine == "" && len(parts) > 0 { + out[len(out)-1].Node = parts[0] + } + } + return out + } + if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) { + for hk := range k.wants { + if node, module := splitNodeModule(hk); module == controllerModule { + out = append(out, about{Node: node, Module: module}) + } + } + } + if s.Machine != "" && s.Scope != health.ScopeMachine { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + } + return out +} + +func splitNodeModule(hk string) (string, string) { + node, module, _ := strings.Cut(hk, ".") + return node, module +} + +// motion is the delivery of module on node, from its want and have. +func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) { + w, wanted := k.wants[node+"."+module] + h, has := k.haves[node+"."+module] + return delivery.MotionOf(w, wanted, h, has, "", false), w +} + +// holdFor says whether key is held now, and as what. Called with mu held. +func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) { + if open || (f == nil && carried == nil) { + return held{}, false + } + var kind, source string + var subject health.Subject + if f != nil { + kind, source, subject = f.Kind, f.Source, f.Subject + } else { + kind, source, subject = carried.Kind, carried.Source, carried.Subject + } + if _, raisedOut := k.outOfHold[key]; raisedOut { + return held{}, false + } + for _, a := range k.aboutOf(kind, source, subject) { + m, w := k.motion(a.Node, a.Module) + if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP { + continue + } + since := k.Now().UTC() + if was, ok := k.held[key]; ok { + since = was.Since + } + return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true + } + return held{}, false +} + +// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once. +func (k *Keeper) keepHeld(h held) error { + k.mu.Lock() + _, was := k.held[h.Key] + k.held[h.Key] = h + bound := k.bound(h.Module) + k.mu.Unlock() + if was { + return nil + } + if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("keeping %s held: %w", h.Key, err) + } + b := boundKey(h.Node, h.Module) + err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version, + "bound-seconds": int(bound / time.Second)}, bound) + if err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err) + } + k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound) + return nil +} + +// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state +// `bounds` is written with it as its own time to live, and its expiry is the bound passing. +func (k *Keeper) bound(string) time.Duration { + if k.Bound > 0 { + return k.Bound + } + return delivery.DefaultDeliveryBound +} + +func boundKey(node, module string) string { return node + "." + module } + +// dropHeld lets a held key go: raised, or gone with why said. +func (k *Keeper) dropHeld(key, why string) { + k.mu.Lock() + h, ok := k.held[key] + delete(k.held, key) + k.mu.Unlock() + if !ok { + return + } + if err := k.Store.Delete(HeldState, key); err != nil { + k.Logf("the held %s could not be removed from held: %v", key, err) + } + if why != "" { + k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why) + } +} + +// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or +// drops what it held. +func (k *Keeper) Want(key string, value []byte, deleted bool) error { + node, module, err := delivery.ParseWantKey(key) + if err != nil { + return nil // not a want of a module on a node (a held fingerprint's key) + } + k.mu.Lock() + k.init() + if deleted { + delete(k.wants, node+"."+module) + } else if w, err := delivery.DecodeWant(value); err == nil { + k.wants[node+"."+module] = w + } else { + k.Logf("the want %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Have takes one change of a node's have. +func (k *Keeper) Have(key string, value []byte, deleted bool) error { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + return nil + } + k.mu.Lock() + k.init() + if deleted { + delete(k.haves, node+"."+module) + } else if h, err := delivery.DecodeHave(value); err == nil { + k.haves[node+"."+module] = h + } else { + k.Logf("the have %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Placement takes one change of the placement: which module holds which seat on a node. +func (k *Keeper) Placement(key string, value []byte, deleted bool) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + return + } + k.mu.Lock() + defer k.mu.Unlock() + k.init() + for hk := range k.holders { + if strings.HasPrefix(hk, node+".") { + delete(k.holders, hk) + } + } + if deleted { + return + } + p, err := delivery.DecodePlacement(value) + if err != nil { + k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err) + return + } + for _, h := range p.Holds { + k.holders[node+"."+h.Claim] = h.Module + } +} + +// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. +func (k *Keeper) BoundPassed(key string) error { + node, module := splitNodeModule(key) + k.mu.Lock() + k.init() + _, w := k.motion(node, module) + holding := false + for _, h := range k.held { + holding = holding || (h.Node == node && h.Module == module) + } + if !holding { + k.mu.Unlock() + return nil + } + k.passed[key] = w.FP + k.mu.Unlock() + return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node, + k.bound(module))) +} + +// judge raises or drops what node and module hold when their delivery ended, or with line past its bound. +func (k *Keeper) judge(node, module, line string) error { + k.mu.Lock() + m, w := k.motion(node, module) + var keys []string + for key, h := range k.held { + if h.Node == node && h.Module == module { + keys = append(keys, key) + } + } + if line == "" && !m.Ended() && m.Delivering { + k.mu.Unlock() + return nil + } + if line == "" { + line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node) + if m.Failed { + line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node) + } + delete(k.passed, node+"."+module) + } + for _, key := range keys { + k.outOfHold[key] = line + } + k.mu.Unlock() + sort.Strings(keys) + var problems []string + for _, key := range keys { + k.dropHeld(key, "") + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + k.mu.Lock() + delete(k.outOfHold, key) + k.mu.Unlock() + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +func short(fp string) string { + if _, rest, ok := strings.Cut(fp, ":"); ok { + fp = rest + } + if len(fp) > 12 { + return fp[:12] + } + return fp +} + +// Held is every key held now, for the read tool and compare. +func (k *Keeper) Held() []held { + k.mu.Lock() + defer k.mu.Unlock() + out := make([]held, 0, len(k.held)) + for _, h := range k.held { + out = append(out, h) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} + +// restoreHeld takes the state `held` as read at the start. +func (k *Keeper) restoreHeld(key string, value []byte) { + var h held + if json.Unmarshal(value, &h) != nil { + return + } + k.mu.Lock() + k.init() + k.held[key] = h + k.mu.Unlock() +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go new file mode 100644 index 00000000..fa668544 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves, +// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases +// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented +// order, not a capture; the values are captured. +const ( + ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c" + ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0" + ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6" + ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142" +) + +// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have. +// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still +// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a +// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a +// held key as held. +func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) { + want := func(fp, composed string) []byte { + raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed}) + return raw + } + have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte { + raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h}) + return raw + } + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"}) + for _, c := range []struct { + name string + // end is the have the delivery ends with, or nil for a delivery past its bound. + end []byte + standing bool + said string + line string + }{ + {name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)}, + {name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK), + standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"}, + {name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"}, + {name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true, + said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"}, + } { + r := newRig() + _ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + // The delivery starts: the assigner wants the next fingerprint; the node applies it. + _ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false) + own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now) + other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now) + machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now) + for _, f := range []struct { + key string + raw []byte + }{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} { + if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil { + t.Fatal(err) + } + } + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + // The delivery's bound is its own key's time to live, the module's bound (the default where it names none). + if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound { + t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"]) + } + if got := r.changes(); got != "raised "+other+", raised "+machine { + t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got) + } + // C8: compare lists the held keys as held, never as a difference. + cmp, _ := r.k.Compare() + held := 0 + for _, d := range cmp.Apart { + if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") { + held++ + } + } + if held != 2 { + t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp) + } + // The carried condition about the controller goes within the delivery; its own finding goes unless it stands. + _ = r.k.Controller("core.controller.deaf", nil, true) + if !c.standing { + _ = r.k.Finding(watchman, own, nil, true) + } + if c.end != nil { + _ = r.k.Have("novox.mesh-controller", c.end, false) + } else { + r.now = r.now.Add(delivery.DefaultDeliveryBound) + _ = r.k.BoundPassed("novox.mesh-controller") + } + if got := r.changes(); got != c.said { + t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said) + } + if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) { + t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation) + } + if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 { + t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held()) + } + } + // A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held + // while the assigner is delivered on ace, though the seat is named mesh-placement. + r := newRig() + placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}}, + Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}}) + if err != nil { + t.Fatal(err) + } + r.k.Placement("placement.ace", placed, false) + _ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now) + _ = r.k.Finding(selfCheck, seat, seatRaw, false) + if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" { + t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held()) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go new file mode 100644 index 00000000..1f0d378c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -0,0 +1,73 @@ +package main + +// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe +// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from +// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the +// lease write, the controller's conditions of the kinds that end with it. +// +// The rows below are what ADR 0564 decides, by the controller's source of each kind as the live controller says it +// (mesh-controller.conditions, read 2026-10-29). **About twenty-five kinds the controller raises have no home in the +// record** (L4.md, "Missing decision"): they have no row here on purpose, and `compare` names each as "no home decided" +// until the lead's decision adds its row. + +// Home is where one condition kind lives. +type Home struct { + // Source is the controller's row or probe that raises it today, for whoever reads the table. + Source string + // Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no + // difference in compare. + Controller bool + // Observer is the module whose findings raise it after the take-over; empty for a kind that ends. + Observer string + // Carried says it ends with the controller's lease and is carried from the controller's bucket until then + // (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller). + Carried bool + // Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in + // compare and not carried. + Replaced bool +} + +// The kinds' homes, by kind. +var Homes = map[string]Home{ + // The watchman's watchdogs (clause 4). + "silent": {Source: "S1", Controller: true, Observer: watchman}, + "max-deliveries": {Source: "S9", Controller: true, Observer: watchman}, + "slow-consumer": {Source: "S9", Controller: true, Observer: watchman}, + "tools-silent": {Source: "S11", Controller: true, Observer: watchman}, + "apply-failing": {Source: "S21", Controller: true, Observer: watchman}, + "module-unhealthy": {Source: "health", Controller: true, Observer: watchman}, + "stuck": {Source: "DM", Controller: true, Observer: watchman}, + // The self-check's probes (clause 6, the Context's table). + "holder-silent": {Source: "D3", Controller: true, Observer: selfCheck}, + "archives-missing": {Source: "D4", Controller: true, Observer: selfCheck}, + "consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck}, + "data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck}, + "agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck}, + "root-not-free": {Source: "D-root", Controller: true, Observer: selfCheck}, + "probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck}, + // The healers' own (clause 7, Limits 2). + "healer-wanted": {Source: "S15", Controller: true, Observer: healers}, + "healers-braked": {Source: "healers", Controller: true, Observer: healers}, + // The kinds that end with the controller and are carried until its lease write (clauses 4, 9). + "controller-deaf": {Source: "S4", Controller: true, Carried: true}, + "lease-lost": {Source: "S12", Controller: true, Carried: true}, + // The kinds that end, their faults observed otherwise (the Context's table). + "core-behind": {Source: "D10", Controller: true, Replaced: true}, +} + +// The observers, as the table names them. +const ( + watchman = "mesh-delivery-watchman" + selfCheck = "mesh-self-check" + healers = "mesh-healers" +) + +// homeOf is the home of a condition's kind, and whether the table has one. +func (k *Keeper) homeOf(kind string) (Home, bool) { + homes := k.Homes + if homes == nil { + homes = Homes + } + h, ok := homes[kind] + return h, ok +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go new file mode 100644 index 00000000..802126f0 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -0,0 +1,665 @@ +package main + +// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the +// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and, +// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From +// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set, +// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition +// but its finding going (ADR 0231 §14). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The module, its seat, and the states it keeps. +const ( + Module = health.KeeperModule + Seat = "mesh-conditions" + + ConditionsState = health.ConditionsState + TakeoverState = health.TakeoverState + HistoryState = "history" + HeldState = "held" + BoundsState = "bounds" + // AskedState keeps the keeper's asks of the operator (go/asker). + AskedState = "asked" +) + +// HistoryKeptFor is how long a transition is kept (clause 3). +const HistoryKeptFor = 90 * 24 * time.Hour + +// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's). +var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule} + +// The other states it reads, as `.`. +const ( + AttemptsRead = health.HealersModule + "." + health.AttemptsState + ControllerConditions = "mesh-controller.conditions" + ControllerLease = "mesh-controller.lease" + ControllerLeaseKey = "holder" + WantsRead = delivery.AssignerModule + ".wants" + // PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the + // lead, 2026-10-29, for ADR 0564 clause 2's inputs). + PlacementRead = delivery.PlacementRead + HavesRead = "node-engine.have" + ConditionWrites = 8 +) + +// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to +// live for a hold's bound. +type Store interface { + Get(state, key string) (value []byte, revision uint64, found bool, err error) + Create(state, key string, value any) (uint64, error) + Update(state, key string, value any, revision uint64) (uint64, error) + Delete(state, key string) error + Keys(state string) ([]string, error) + CreateTTL(state, key string, value any, ttl time.Duration) error +} + +// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)). +type held struct { + Key string `json:"key"` + Node string `json:"node"` + Module string `json:"module"` + Since time.Time `json:"since"` + Version string `json:"version"` + Wanted json.RawMessage `json:"-"` +} + +// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening. +type clearedMemo struct { + at time.Time + count int + began time.Time + gaps []health.Gap + silenced *health.Silence + tried []health.Tried +} + +// Keeper is the condition keeper's state and its writes. +type Keeper struct { + Store Store + // Say publishes one event on the seat; History keeps one transition. + Say func(e health.Event) error + Now func() time.Time + Logf func(format string, a ...any) + // Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil. + Homes map[string]Home + // Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go). + Bound time.Duration + + mu sync.Mutex + findings map[string]map[string]health.Finding + blind map[string]string + attempts map[string]health.Attempt + ctrl map[string]health.Condition + leaseOut bool + takeover health.Takeover + wants map[string]delivery.Want + haves map[string]delivery.Have + // holders are, by `.`, the module the placement says holds the seat there. + holders map[string]string + stuck map[string]string + held map[string]held + // passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the + // line a key raised out of a hold says (hold.go). + passed map[string]string + outOfHold map[string]string + cleared map[string]clearedMemo + seq uint64 +} + +func (k *Keeper) init() { + if k.findings == nil { + k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{}, + map[string]health.Attempt{}, map[string]health.Condition{} + k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{}, + map[string]string{}, map[string]held{}, map[string]clearedMemo{} + k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController} + k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{} + } + if k.Now == nil { + k.Now = time.Now + } + if k.Logf == nil { + k.Logf = func(string, ...any) {} + } +} + +// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears +// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was. +func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if k.findings[observer] == nil { + k.findings[observer] = map[string]health.Finding{} + } + if deleted { + delete(k.findings[observer], key) + } else { + f, err := health.DecodeFinding(value) + if err != nil { + k.mu.Unlock() + k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err) + return nil + } + if f.Key != key { + k.mu.Unlock() + k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key) + return nil + } + k.findings[observer][key] = f + } + k.mu.Unlock() + return k.Settle(key) +} + +// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing +// (ADR 0227 §4). Empty why says it is read again. +func (k *Keeper) Blind(observer, why string) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + if why == "" { + delete(k.blind, observer) + } else { + k.blind[observer] = why + } +} + +// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation +// (clause 7). +func (k *Keeper) Attempt(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + var condition string + if old, ok := k.attempts[key]; ok { + condition = old.Condition + } + if deleted { + delete(k.attempts, key) + } else { + var a health.Attempt + if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" { + k.mu.Unlock() + k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err) + return nil + } + k.attempts[key], condition = a, a.Condition + } + k.mu.Unlock() + if condition == "" { + return nil + } + return k.Settle(condition) +} + +// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write. +func (k *Keeper) Controller(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if deleted { + delete(k.ctrl, key) + } else { + var c health.Condition + if err := json.Unmarshal(value, &c); err != nil { + k.mu.Unlock() + k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err) + return nil + } + k.ctrl[key] = c + } + k.mu.Unlock() + return k.Settle(key) +} + +// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no +// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear. +func (k *Keeper) Lease(away bool) error { + k.mu.Lock() + k.init() + was := k.leaseOut + k.leaseOut = away + keys := make([]string, 0, len(k.ctrl)) + for key := range k.ctrl { + keys = append(keys, key) + } + k.mu.Unlock() + if was == away { + return nil + } + return k.settleAll(keys) +} + +// LeaseAway says whether the controller's lease is handed away. +func (k *Keeper) LeaseAway() bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.leaseOut +} + +// Takeover takes the keeper's own take-over key as read. +func (k *Keeper) Takeover(t health.Takeover) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + k.takeover = t +} + +// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves +// it under this one. +func (k *Keeper) Settle(key string) error { + var err error + for i := 0; i < ConditionWrites; i++ { + if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err) +} + +func (k *Keeper) settleAll(keys []string) error { + sort.Strings(keys) + var problems []string + for _, key := range keys { + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's +// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it. +func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) { + observers := make([]string, 0, len(k.findings)) + for o := range k.findings { + observers = append(observers, o) + } + sort.Strings(observers) + for _, o := range observers { + if f, ok := k.findings[o][key]; ok { + return &f, nil + } + } + if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) { + return nil, &c + } + return nil, nil +} + +func (k *Keeper) endsWithController(c health.Condition) bool { + h, ok := k.homeOf(c.Kind) + return ok && h.Carried +} + +// settleOnce is one try of Settle. +func (k *Keeper) settleOnce(key string) error { + now := k.Now().UTC() + raw, rev, open, err := k.Store.Get(ConditionsState, key) + if err != nil { + return fmt.Errorf("reading the condition %s: %w", key, err) + } + var was health.Condition + if open { + if err := json.Unmarshal(raw, &was); err != nil { + return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + } + k.mu.Lock() + k.init() + f, carried := k.wanted(key) + hold, holding := k.holdFor(key, f, carried, open) + k.mu.Unlock() + + if f == nil && carried == nil { + k.dropHeld(key, "its finding went while its module was delivered") + if !open { + return nil + } + return k.clear(key, was, rev, now, "its finding went") + } + if holding { + return k.keepHeld(hold) + } + k.dropHeld(key, "") + next := k.derive(key, was, open, f, carried, now) + events := transitions(was, next, open) + if len(events) == 0 && open && sameBody(was, next) { + return nil + } + if open { + _, err = k.Store.Update(ConditionsState, key, next, rev) + } else { + _, err = k.Store.Create(ConditionsState, key, next) + } + if err != nil { + return err + } + k.mu.Lock() + delete(k.cleared, key) + k.mu.Unlock() + for _, e := range events { + e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change) + e.Show = k.show(next, carried != nil) + k.tell(e) + } + return nil +} + +func (k *Keeper) show(c health.Condition, carried bool) string { + if carried { + return health.ShowController(c.Key) + } + return health.ShowKeeper(c.Key) +} + +// derive is the condition key should be now, from what was open and what the inputs say. +func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition { + var c health.Condition + switch { + case carried != nil: + // Raised and cleared as the controller's bucket says, with its show and actions (Limits 9). + c = *carried + c.Epoch = 0 + c.LooksUnread = "" + case open: + c = was + k.fromFinding(&c, *f, now) + default: + c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf} + k.fromFinding(&c, *f, now) + k.mu.Lock() + if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin { + c.Count = before.count + 1 + c.First = before.began + c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now}) + if over := len(c.Gaps) - health.KeptGaps; over > 0 { + c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:] + } + if before.silenced != nil && now.Before(before.silenced.Until) { + c.Silenced = before.silenced + } + c.Tried = before.tried + } + k.mu.Unlock() + } + if carried == nil { + k.overlay(&c, now) + } + if !open { + k.mu.Lock() + if line := k.outOfHold[key]; line != "" { + c.Explanation = strings.TrimSpace(c.Explanation + " " + line) + } + k.mu.Unlock() + } + if c.Silenced != nil && !now.Before(c.Silenced.Until) { + c.Silenced = nil + } + return c +} + +// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence +// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again. +func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) { + escalated := c.Escalated() + c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source + c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions + if !(escalated && f.Severity != health.Urgent) { + c.Severity = f.Severity + } + if !escalated { + c.Resolver = f.Resolver + if c.Resolver == "" { + c.Resolver = health.ResolverSelf + } + } + if len(f.Evidence) > 0 { + c.Evidence = append([]health.Evidence(nil), f.Evidence...) + c.LastObserved = f.Evidence[0].At + } else if c.Evidence == nil { + c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}} + c.LastObserved = f.At + } + c.Observations = len(c.Evidence) + if c.Actions == nil { + c.Actions = []health.Action{} + } + _ = now +} + +// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's +// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has +// none of its own. +func (k *Keeper) overlay(c *health.Condition, now time.Time) { + k.mu.Lock() + var mine []health.Attempt + for _, a := range k.attempts { + if a.Condition == c.Key { + mine = append(mine, a) + } + } + ctrl, carry := k.ctrl[c.Key] + carry = carry && !k.leaseOut + k.mu.Unlock() + sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) }) + if carry && len(ctrl.Tried) > 0 { + c.Tried = append([]health.Tried(nil), ctrl.Tried...) + if ctrl.Escalated() { + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + } else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf { + c.Resolver = ctrl.Resolver + } + } + for _, a := range mine { + t := a.Tried() + if !containsTried(c.Tried, t) { + c.Tried = append(c.Tried, t) + } + switch a.Outcome { + case health.OutcomeEscalated: + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + default: + if c.Resolver != health.ResolverOperator { + c.Resolver = health.ResolverHealer(a.Healer) + } + } + } + if len(c.Tried) > health.KeptTried { + c.Tried = c.Tried[len(c.Tried)-health.KeptTried:] + } + if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) { + c.Silenced = ctrl.Silenced + } +} + +func containsTried(list []health.Tried, t health.Tried) bool { + for _, x := range list { + if x.At.Equal(t.At) && x.By == t.By && x.What == t.What { + return true + } + } + return false +} + +// transitions are the events between what was open and what is now, in the controller's order: raised or reopened; +// else severity, resolver, silence and words changes. An observation that changes none of these says nothing. +func transitions(was, now health.Condition, open bool) []health.Event { + if !open { + change := health.ChangeRaised + if now.Count > 1 { + change = health.ChangeReopened + } + return []health.Event{{Change: change}} + } + var out []health.Event + if was.Severity != now.Severity { + out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)}) + } + if was.Resolver != now.Resolver { + out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver}) + } + switch { + case was.Silenced == nil && now.Silenced != nil: + out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why}) + case was.Silenced != nil && now.Silenced == nil: + out = append(out, health.Event{Change: health.ChangeUnsilenced}) + } + if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) { + out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs}) + } + return out +} + +func sameActions(a, b []health.Action) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) || (len(a) == 0 && len(b) == 0) +} + +func sameBody(a, b health.Condition) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// clear removes an open condition whose finding went, and says it (clause 2). +func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error { + // A delete at the revision read: a write that moved it meanwhile is read again. + if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil { + return err + } + if err := k.Store.Delete(ConditionsState, key); err != nil { + return err + } + k.mu.Lock() + k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried} + k.mu.Unlock() + cleared := now + k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why, + Cleared: &cleared, Show: k.show(was, false)}) + return nil +} + +// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition +// and its history still hold it. +func (k *Keeper) tell(e health.Event) { + k.mu.Lock() + k.seq++ + hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq) + k.mu.Unlock() + if _, err := k.Store.Create(HistoryState, hk, e); err != nil { + k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err) + } + if k.Say == nil { + return + } + if err := k.Say(e); err != nil { + k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err) + } +} + +// Open is every open condition, ordered as `conditions` says them. +func (k *Keeper) Open() ([]health.Condition, error) { + keys, err := k.Store.Keys(ConditionsState) + if err != nil { + return nil, err + } + var out []health.Condition + for _, key := range keys { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + continue + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + out = append(out, c) + } + health.Order(out) + return out, nil +} + +// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared. +func (k *Keeper) Resync() (int, error) { + open, err := k.Open() + if err != nil { + return 0, err + } + seen := map[string]bool{} + for _, c := range open { + seen[c.Key] = true + } + k.mu.Lock() + k.init() + for o, fs := range k.findings { + if k.blind[o] != "" { + continue + } + for key := range fs { + seen[key] = true + } + } + for key := range k.ctrl { + seen[key] = true + } + k.mu.Unlock() + keys := make([]string, 0, len(seen)) + for key := range seen { + keys = append(keys, key) + } + return len(keys), k.settleAll(keys) +} + +// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in +// history, naming who and why. +func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) { + if strings.TrimSpace(why) == "" { + return health.Condition{}, errors.New("a silence says why") + } + if d <= 0 || d > health.MaxSilence { + return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d) + } + for i := 0; i < ConditionWrites; i++ { + raw, rev, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return health.Condition{}, err + } + if !found || string(raw) == "null" { + return health.Condition{}, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return health.Condition{}, err + } + now := k.Now().UTC() + c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now} + if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) { + continue + } else if err != nil { + return health.Condition{}, err + } + k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced, + Why: why + " (" + by + ")", Show: health.ShowKeeper(key)}) + return c, nil + } + return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key) +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go new file mode 100644 index 00000000..cc90e26a --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -0,0 +1,314 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// fakeStore is the bus's states in memory, with revisions, as the runtime answers them. +type fakeStore struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + next uint64 + // ttls are the times to live the keys were created with. + ttls map[string]time.Duration +} + +func newFakeStore() *fakeStore { + return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}} +} + +func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.vals[state+"/"+key] + return v, f.revs[state+"/"+key], ok, nil +} + +func (f *fakeStore) put(state, key string, value any) uint64 { + raw, _ := json.Marshal(value) + f.next++ + f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next + return f.next +} + +func (f *fakeStore) Create(state, key string, value any) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.vals[state+"/"+key]; ok { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.revs[state+"/"+key] != rev { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Delete(state, key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.vals, state+"/"+key) + delete(f.revs, state+"/"+key) + return nil +} + +func (f *fakeStore) Keys(state string) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.vals { + if s, key, _ := strings.Cut(k, "/"); s == state { + out = append(out, key) + } + } + sort.Strings(out) + return out, nil +} + +func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + f.mu.Lock() + if f.ttls == nil { + f.ttls = map[string]time.Duration{} + } + f.ttls[state+"/"+key] = ttl + f.mu.Unlock() + _, err := f.Create(state, key, value) + return err +} + +// rig is a keeper over a fake store, its clock and what it said. +type rig struct { + k *Keeper + st *fakeStore + now time.Time + said []health.Event +} + +func newRig() *rig { + r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)} + r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error { + r.said = append(r.said, e) + return nil + }} + return r +} + +func (r *rig) changes() string { + var out []string + for _, e := range r.said { + out = append(out, e.Change+" "+e.Key) + } + r.said = nil + return strings.Join(out, ", ") +} + +func (r *rig) open(key string) (health.Condition, bool) { + raw, _, ok, _ := r.st.Get(ConditionsState, key) + var c health.Condition + if ok { + _ = json.Unmarshal(raw, &c) + } + return c, ok +} + +// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding). +func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) { + t.Helper() + f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id}, + Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source, + Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + return f.Key, raw +} + +// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer +// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten +// minutes is the same fault; a silence is a transition in history. +func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) { + r := newRig() + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + steps := []struct { + name string + do func() error + said string + check func(c health.Condition, open bool) string + }{ + {name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) }, + said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string { + if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }}, + {name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err }, + check: func(c health.Condition, open bool) string { + if !open { + return "cleared by blindness" + } + return "" + }}, + {name: "a healer's attempt reaches tried and the resolver", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now}) + return r.k.Attempt("h3-1", a, false) + }, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "an escalation makes it urgent, the operator's", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated, + Begun: r.now.Add(time.Minute)}) + return r.k.Attempt("h3-2", a, false) + }, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "a silence is a transition with who and why", do: func() error { + _, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable") + return err + }, said: "silenced machine.g14.silent"}, + {name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) }, + said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string { + if open { + return "still open" + } + return "" + }}, + {name: "raised again within ten minutes is a reopening", do: func() error { + r.now = r.now.Add(5 * time.Minute) + return r.k.Finding(watchman, key, raw, false) + }, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Count != 2 || len(c.Gaps) != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + // Until the lease write the controller's escalation for the same key reaches the condition (clause 8). + {name: "the controller's escalation is carried", do: func() error { + ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator, + Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}}) + _ = r.k.Attempt("h3-1", nil, true) + _ = r.k.Attempt("h3-2", nil, true) + return r.k.Controller(key, ctrl, false) + }, check: func(c health.Condition, _ bool) string { + if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + } + for _, s := range steps { + if err := s.do(); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" { + t.Errorf("%s: said %q, want %q", s.name, got, s.said) + } + if s.check != nil { + c, open := r.open(key) + if why := s.check(c, open); why != "" { + t.Errorf("%s: %s", s.name, why) + } + } + } + // Every transition is kept in history, and each event carries the keeper's show. + keys, _ := r.st.Keys(HistoryState) + if len(keys) < 7 { + t.Errorf("history keeps %d transitions", len(keys)) + } + raw0, _, _, _ := r.st.Get(HistoryState, keys[0]) + if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) { + t.Errorf("the first transition is %s", raw0) + } + // A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the + // lease is handed away, with the controller's show (clause 9, Limits 9). + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9}) + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" { + t.Errorf("a carried kind is %+v, said %+v", c, r.said) + } + r.said = nil + if err := r.k.Lease(true); err != nil { + t.Fatal(err) + } + if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" { + t.Error("a carried kind outlived the lease write") + } +} + +// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held +// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed +// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and +// hand-back after the lease write. +func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) { + r := newRig() + put := func(key, kind string) []byte { + raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"}) + return raw + } + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + _ = r.k.Finding(watchman, key, raw, false) + _ = r.k.Controller(key, put(key, "silent"), false) + only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now) + _ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false) + _ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false) + _ = r.k.Controller("delivery.x.stalled", put("delivery.x.stalled", "stalled"), false) + _ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false) + cmp, err := r.k.Compare() + if err != nil { + t.Fatal(err) + } + reasons := map[string]string{} + for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) { + reasons[d.Key] = d.Reason + } + for key, says := range map[string]string{ + only: "keeper only", + "core.controller.deaf": "carried", + "delivery.x.stalled": "no home is decided for the kind stalled", + "seat.vault.ace.holder-silent": "mesh-self-check has not written its finding", + } { + if !strings.Contains(reasons[key], says) { + t.Errorf("%s: %q, want it to say %q", key, reasons[key], says) + } + } + if cmp.Same != 1 || len(cmp.Differences) != 2 { + t.Errorf("compare: %+v", cmp) + } + if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") { + t.Errorf("take-over while compare differs: %v", err) + } + if _, err := r.k.HandBack("jochen", "back"); err != nil { + t.Errorf("hand-back before the lease write: %v", err) + } + _ = r.k.Lease(true) + if _, err := r.k.HandBack("jochen", "back"); err == nil { + t.Error("hand-back after the lease write was taken") + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go new file mode 100644 index 00000000..ac597d4b --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -0,0 +1,164 @@ +// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A +// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its +// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the +// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's +// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the +// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP +// channel; what this module says, it says on stderr. +package main + +import ( + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} + +func (runtimeStore) Create(state, key string, value any) (uint64, error) { + return stdio.State(state).Create(key, value) +} + +func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) { + return stdio.State(state).Update(key, value, revision) +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire. +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) + return err +} + +// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3). +func sayOnSeat(e health.Event) error { + return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "") +} + +// NewKeeper is the keeper over the runtime. +func NewKeeper() *Keeper { + return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf} +} + +// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not +// delivered) is read again when it is. +func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) { + go func() { + for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) { + err := stdio.State(state).Watch("", handle) + if err == nil { + if observer != "" { + k.Blind(observer, "") + } + return + } + if observer != "" { + k.Blind(observer, err.Error()) + } + logf("%s cannot be watched (%v); watched again in %s", state, err, wait) + time.Sleep(wait) + } + }() +} + +// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13). +func Keep(k *Keeper) { + for _, s := range []string{HeldState} { + keys, _ := stdio.State(s).Keys() + for _, key := range keys { + if e, err := stdio.State(s).Get(key); err == nil && e != nil { + k.restoreHeld(key, e.Value) + } + } + } + if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil { + var raw []byte + if e != nil { + raw = e.Value + } + if t, err := health.DecodeTakeover(raw, e != nil); err == nil { + k.Takeover(t) + } + } + said := func(err error) error { + if err != nil { + logf("%v", err) + } + return nil + } + watch(k, ControllerLease, "", func(c stdio.StateChange) error { + if c.Key != ControllerLeaseKey { + return nil + } + away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil) + logf("the controller's lease: %s", why) + return said(k.Lease(away)) + }) + watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) }) + watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) }) + watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error { + return said(k.Attempt(c.Key, c.Value, c.Op != "put")) + }) + watch(k, ControllerConditions, "", func(c stdio.StateChange) error { + if k.LeaseAway() { + return nil + } + return said(k.Controller(c.Key, c.Value, c.Op != "put")) + }) + for _, o := range Observers { + watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error { + return said(k.Finding(o, c.Key, c.Value, c.Op != "put")) + }) + } + watch(k, BoundsState, "", func(c stdio.StateChange) error { + if c.Op == stdio.OpExpire { + return said(k.BoundPassed(c.Key)) + } + return nil + }) + watch(k, PlacementRead, "", func(c stdio.StateChange) error { + k.Placement(c.Key, c.Value, c.Op != stdio.OpPut) + return nil + }) +} + +func main() { + k := NewKeeper() + client := asker.New(AskedState) + client.Module = Module + go func() { + <-stdio.Served() + Keep(k) + if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) { + if err != nil { + logf("the answer to %s (%s): %v", w.Ask, h, err) + } + }); err != nil { + logf("the operator's answers cannot be heard: %v", err) + } + }() + if err := stdio.Serve("", Tools(k, client)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go new file mode 100644 index 00000000..94b1801c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat +// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers' +// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its +// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it, +// every act a tool that starts it. +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("the manifest is refused: %v", err) + } + seat, _ := catalogue.SeatNamed(Seat) + if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat || + !slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) { + t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves)) + } + reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead} + for _, o := range Observers { + reads = append(reads, o+"."+health.FindingsState) + } + slices.Sort(reads) + got := slices.Clone(m.Reads) + slices.Sort(got) + if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) { + t.Errorf("reads %v, want %v and never the healers' would", got, reads) + } + ttl, keyTTL := map[string]int{}, map[string]bool{} + for _, s := range m.State { + ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL + } + // Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age. + if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] { + t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v", + ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState]) + } + // Clause 12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool, + HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync", + "what it reads": InputsTool} + tools := map[string]bool{} + for _, tool := range Tools(&Keeper{}, nil) { + tools[tool.Name] = true + } + for name := range ttl { + if readBy[name] == "" { + t.Errorf("the state %s has no tool that reads it", name) + } + } + for what, tool := range readBy { + if !tools[tool] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/reads-outside b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside new file mode 100644 index 00000000..c20a67a5 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-conditions/module.json diff --git a/modules/mesh-conditions/cmd/mesh-conditions/tools.go b/modules/mesh-conditions/cmd/mesh-conditions/tools.go new file mode 100644 index 00000000..330e153e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/tools.go @@ -0,0 +1,252 @@ +package main + +// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions` +// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the +// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call. + +import ( + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"} + +// The keeper's own read tools. +const ( + HistoryTool = "conditions_history" + HeldTool = "conditions_held" + TakeoverTool = "conditions_takeover" + InputsTool = "conditions_inputs" +) + +// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314). +const briefEvidence = 1 + +// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole. +type inBrief struct { + health.Condition + EvidenceKept int `json:"evidence-kept,omitempty"` + More string `json:"more,omitempty"` +} + +func obj(props map[string]any, required ...string) map[string]any { + out := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + out["required"] = required + } + return out +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +// Tools are what the runtime serves. +func Tools(k *Keeper, a Asks) []stdio.Tool { + return []stdio.Tool{ + {Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " + + "oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " + + "one whole; with history, every transition lately.", + Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"), + "severity": str("urgent or warning"), "machine": str("only those about this machine"), + "history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}), + Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }}, + {Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " + + "transition in history; it stays open.", + Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"), + "why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"), + Run: func(args map[string]any) (any, error) { + d, err := duration(strArg(args, "for")) + if err != nil { + return nil, err + } + return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " + + "its reason; what ends with the controller, what only the keeper raises and what is held listed apart.", + Run: func(map[string]any) (any, error) { return k.Compare() }}, + {Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " + + "keeper; refused while compare differs.", + Input: obj(map[string]any{"why": str("why now")}, "why"), + Run: func(args map[string]any) (any, error) { + return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " + + "is handed away.", + Input: obj(map[string]any{"why": str("why")}, "why"), + Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }}, + {Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.", + Run: func(map[string]any) (any, error) { + n, err := k.Resync() + return map[string]any{"settled": n}, err + }}, + {Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.", + Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}), + Run: func(args map[string]any) (any, error) { + return k.History(strArg(args, "key"), intArg(args, "limit", 50)) + }}, + {Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " + + "with its module, node, version and since when, and the deliveries' bounds.", + Run: func(map[string]any) (any, error) { + bounds, err := k.Store.Keys(BoundsState) + return map[string]any{"held": k.Held(), "bounds": bounds}, err + }}, + {Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.", + Run: func(map[string]any) (any, error) { + raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return nil, err + } + return health.DecodeTakeover(raw, found) + }}, + {Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " + + "cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.", + Run: func(map[string]any) (any, error) { return k.Inputs(), nil }}, + } +} + +// ConditionsVerb is `conditions`: the list, one whole, or the history. +func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) { + key := strArg(args, "key") + if strArg(args, "history") == "true" { + days := intArg(args, "days", 7) + if days < 1 || days > 90 { + return nil, fmt.Errorf("days is from 1 to 90, not %d", days) + } + return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days) + } + if key != "" { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + return nil, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + return c, json.Unmarshal(raw, &c) + } + scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine") + if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) { + return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity) + } + open, err := k.Open() + if err != nil { + return nil, err + } + list := []inBrief{} + counted := map[string]int{} + for _, c := range open { + if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) || + (machine != "" && !concerns(c, machine)) { + continue + } + b := inBrief{Condition: c} + if len(c.Evidence) > briefEvidence { + b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key) + } + list = append(list, b) + counted[c.Source+" "+c.Kind]++ + } + return map[string]any{"conditions": list, "open": len(open), "counted": counted, + "note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " + + "its newest evidence — `conditions key=` gives one whole"}, nil +} + +// concerns says whether a condition is about a machine: it names it, or its key does. +func concerns(c health.Condition, machine string) bool { + return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || + slices.Contains(strings.Split(c.Key, "."), machine) +} + +// History is the newest transitions, at most limit, of one key or all. +func (k *Keeper) History(key string, limit int) ([]health.Event, error) { + all, err := k.HistorySince(key, time.Time{}, 0) + if err != nil { + return nil, err + } + events := all["history"].([]health.Event) + slices.Reverse(events) + if len(events) > limit { + events = events[:limit] + } + return events, nil +} + +// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it. +func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) { + keys, err := k.Store.Keys(HistoryState) + if err != nil { + return nil, err + } + events := []health.Event{} + for _, hk := range keys { + raw, _, found, err := k.Store.Get(HistoryState, hk) + if err != nil || !found { + continue + } + var e health.Event + if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) { + continue + } + events = append(events, e) + } + return map[string]any{"history": events, "days": days}, nil +} + +// Inputs is what the keeper reads, as it stands. +func (k *Keeper) Inputs() map[string]any { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + findings := map[string]int{} + for o, fs := range k.findings { + findings[o] = len(fs) + } + return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts), + "controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow, + "wants": len(k.wants), "haves": len(k.haves)} +} + +func strArg(a map[string]any, name string) string { + switch v := a[name].(type) { + case string: + return strings.TrimSpace(v) + case bool: + return strconv.FormatBool(v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + } + return "" +} + +func intArg(a map[string]any, name string, def int) int { + if n, err := strconv.Atoi(strArg(a, name)); err == nil { + return n + } + return def +} + +// duration reads 30m, 4h or 2d. +func duration(s string) (time.Duration, error) { + if n, ok := strings.CutSuffix(s, "d"); ok { + days, err := strconv.Atoi(n) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return time.Duration(days) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return d, nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go new file mode 100644 index 00000000..f8881730 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -0,0 +1,335 @@ +//go:build integration + +package main + +// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary, +// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a +// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its +// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as +// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as +// condition-cleared. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the keeper needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is a state's bucket: the keeper's own by its local name, another's by `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, read := strings.Cut(state, "."); read { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + kv, kvErr := p.kv(a.State) + switch m.Method { + case "mesh/state.get": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.create", "mesh/state.update", "mesh/state.put": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + var rev uint64 + var err error + switch m.Method { + case "mesh/state.create": + rev, err = kv.Create(p.ctx, a.Key, a.Value) + case "mesh/state.update": + rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision) + default: + rev, err = kv.Put(p.ctx, a.Key, a.Value) + } + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}}) + return + } + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.keys": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + err := kvErr + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watchID string + _ = json.Unmarshal(m.ID, &watchID) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID, + "change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())), + "revision": e.Revision(), "current": current}}}) + } + case "mesh/seat.publish": + p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body)) + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func nullIfEmpty(b []byte) []byte { + if len(b) == 0 { + return []byte("null") + } + return b +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears +// it. +func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the keeper does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement", + "mesh-healers_attempts"} + for _, o := range Observers { + buckets = append(buckets, o+"_findings") + } + for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} { + buckets = append(buckets, Module+"_"+s) + } + for _, b := range buckets { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>") + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + // The first call serves; the watches start after it. + if _, err := p.call(InputsTool, map[string]any{}); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + at := time.Now().UTC() + f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14", + Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline", + Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1", + Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings") + if _, err := findings.Put(ctx, f.Key, raw); err != nil { + t.Fatal(err) + } + msg, err := events.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("no condition-raised was said: %v", err) + } + var e health.Event + if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" || + e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline { + t.Fatalf("said %s on %s", msg.Data, msg.Subject) + } + listed, err := p.call(Seat+".conditions", map[string]any{}) + if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) { + t.Fatalf("conditions answered %s, %v", listed, err) + } + if err := findings.Delete(ctx, f.Key); err != nil { + t.Fatal(err) + } + msg, err = events.NextMsg(10 * time.Second) + if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" { + t.Fatalf("no condition-cleared was said: %v", err) + } + conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState) + if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the cleared condition is still kept: %v", err) + } +} diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod new file mode 100644 index 00000000..23a7dffb --- /dev/null +++ b/modules/mesh-conditions/go.mod @@ -0,0 +1,22 @@ +module mesh-conditions + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-conditions/go.sum b/modules/mesh-conditions/go.sum new file mode 100644 index 00000000..3ecc6a80 --- /dev/null +++ b/modules/mesh-conditions/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json new file mode 100644 index 00000000..89354c38 --- /dev/null +++ b/modules/mesh-conditions/module.json @@ -0,0 +1,84 @@ +{ + "module": "mesh-conditions", + "version": "1", + "runs-as": "mesh-conditions", + "claims": [ + { + "name": "mesh-conditions", + "scope": "mesh", + "serves": [ + "conditions", + "silence", + "compare", + "take-over", + "hand-back", + "resync" + ] + } + ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-watchman.findings", + "mesh-self-check.findings", + "mesh-healers.findings", + "mesh-delivery-assigner.findings", + "mesh-healers.attempts", + "mesh-controller.conditions", + "mesh-controller.lease", + "mesh-delivery-assigner.wants", + "node-engine.have", + "mesh-delivery-assigner.placement" + ], + "state": [ + "conditions", + { + "name": "history", + "ttl-seconds": 7776000 + }, + "takeover", + "held", + { + "name": "bounds", + "key-ttl": true + }, + "asked" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-conditions", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-conditions", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-conditions" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-conditions", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-conditions", + "binary": "mesh-conditions", + "loads": [ + "mesh-conditions" + ] + } + ] + } +} diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md index 42bd1ed0..72220615 100644 --- a/modules/mesh-config/README.md +++ b/modules/mesh-config/README.md @@ -61,3 +61,21 @@ them, newest first; with `proposal`, one read whole, with whether the router's a Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## The switch from the controller's records (novox/hq ADR 0565 §3, §4, §13) + +- `mesh-config_declared`: each placed module's declarations at the trunk, with the manifest and commit they come + from, written only when they differ. +- `mesh-config_takeover`: the key `switch` names the source that holds: none (the interim since mesh-catalog #558: + declarations from the modules' own manifests, the list of predefined keys and pull requests' config from the + controller), `indexer` after the operator's take-over (the list and pull requests' config from the indexer's + `predefined` and `config-candidates`; the controller's `pull-declares` refused), `controller` after a hand-back + (declarations, list and pull requests' config from the controller's records). Each take-over ask is kept beside it. +- `config_compare` lists every difference between the own sources and the controller's records: each module's + declarations, the list, each open pull request's config, and every placed module's value on every node resolved + both ways. `config_take_over` asks the operator's Approve, refused while any difference stands and switched on the + Approve only if none stands then. `config_hand_back` returns to the controller's records until its lease is handed + away. `config_declared` and `config_resync` read and write again what it keeps. +- From the controller's lease write a trusted value is set only through propose and the operator's Approve. +- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the + controller's while that seat has no holder and the controller holds its lease. diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index 0f2435cc..1928a3df 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level, strings.Join(problems, "; ")) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+ "the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+ "propose it instead, with config's action propose", call.Key, call.Caller) @@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { if !had { return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+ "operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+ "config's action propose and clear", call.Key, call.Caller) @@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return answer, nil } +// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is +// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever +// calls (novox/hq ADR 0565 §13). +func (c *Config) atTheTerminal(call Call) bool { + return call.Caller == ControllerCaller && !c.leaseAway() +} + // declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared // keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10). func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) { diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 036acd86..c0bba92e 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -19,6 +19,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. @@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) { func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) } -// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb. +// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat +// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11). +var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool { + e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +}} + +// HandAct records a change in the hand-act log (to-be 45 §7). func (runtimeStore) HandAct(what, why, cause string) error { - _, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause}) + _, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause}) return err } @@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error { func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) } func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() } func (runtimeAsks) Warranted(ask string) error { - _, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask}) + _, err := recorder.RecordWarrant(Module, ask) return err } @@ -118,7 +132,8 @@ func resolving(c *Config) { } // A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is // every change of what a module declares where (issue 1953); the builds themselves are read, not watched. - for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} { + for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead, + CandidatesRead, MirrorDeclaredRead, TakeoverState} { go func(read string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { time.Sleep(wait) @@ -221,9 +236,9 @@ func hearing(c *Config) { } } -// Tools is the seat's verb as the runtime serves it. +// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them. func Tools(c *Config) []stdio.Tool { - return []stdio.Tool{{ + return append(switchTools(c), stdio.Tool{ Name: Seat + "." + Verb, Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + @@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool { call.Caller = stdio.CallerOf(args) return c.Do(call) }, - }} + }) } // callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go new file mode 100644 index 00000000..c68914e4 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself +// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools. +func TestEveryStateAndActHasItsTool(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + State []string `json:"state"` + Logs []string `json:"logs"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + via := map[string]string{ + ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list", + ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history", + DeclaredState: DeclaredTool, TakeoverState: DeclaredTool, + PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool, + ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool, + IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool, + "a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool, + } + served := map[string]bool{} + for _, tool := range Tools(&Config{}) { + served[tool.Name] = true + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") { + t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs) + } + for _, verb := range SeatVerbs { + if !served[Seat+"."+verb] { + t.Errorf("the seat's verb %s is not served", verb) + } + } + for _, name := range m.Tools { + if !served[name] { + t.Errorf("the manifest lists the tool %s, which is not served", name) + } + } + for _, list := range [][]string{m.State, m.Logs, m.Reads} { + for _, s := range list { + if _, ok := via[s]; !ok { + t.Errorf("%s has no tool that reads it", s) + } + } + } + for what, tool := range via { + name, _, _ := strings.Cut(tool, " ") + if name == "config" { + name = Seat + "." + Verb + } + if !served[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index d6cb304d..18619024 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) { for _, r := range ready { said = append(said, c.decide(r.p, r.w)) } + said = append(said, c.decideTakeovers()...) c.recordWarrants(now) return said, nil } diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go index f803b5de..8c0647a6 100644 --- a/modules/mesh-config/cmd/mesh-config/pulls.go +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -10,6 +10,8 @@ import ( "strconv" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826). @@ -61,31 +63,22 @@ type PullDecl struct { At time.Time `json:"at"` } -// KeptValue is an approved held value, kept apart from the values mesh-config resolves. -type KeptValue struct { - Pull string `json:"pull"` - Head string `json:"head"` - Digest string `json:"declaration"` - Node string `json:"node"` - Module string `json:"module"` - Key string `json:"key"` - Value json.RawMessage `json:"value"` - By string `json:"by"` - Via string `json:"via"` - Why string `json:"why"` - // Approved is who approved it, as the proposal kept it. - Approved string `json:"approved"` - Proposal string `json:"proposal"` - At time.Time `json:"at"` -} +// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the +// checker and the assigner's composition check read (novox/hq ADR 0565 §5). +type KeptValue = delivery.KeptValue +// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is +// not one. func keptKey(pull, node, module, key string) string { + if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil { + return k + } return pullKey(pull) + "." + node + "." + module + "." + key } -func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } +func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } -func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } +func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } // declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same. func declDigest(d Declaration) string { @@ -151,6 +144,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+ "mesh-config what the head declares; %s is not the controller", call.Caller) } + if sw, err := c.source(); err != nil { + return nil, err + } else if sw.Source == SourceIndexer { + return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " + + "declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)") + } if _, _, _, err := parsePull(call.Pull); err != nil { return nil, err } @@ -185,6 +184,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { pull := strings.TrimSpace(call.Pull) c.mu.Lock() defer c.mu.Unlock() + return c.declarePull(pull, head, declares) +} + +// declarePull keeps what a pull request's head declares and judges again the values held for it; from the +// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock. +func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) { if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares, At: c.now()}); err != nil { return nil, fmt.Errorf("the declarations were not kept: %v", err) @@ -235,7 +240,7 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { // heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and // config-changed; what could not be done is answered, not dropped. func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string { - level := kv.level() + level := keptLevel(kv) ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why, Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}} unsaid := c.record(level.ValueKey(kv.Key), ch) @@ -252,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string // dropKept removes a held value, saying why. func (c *Config) dropKept(kv KeptValue, why string) []string { - if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil { - return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()} + if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil { + return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()} } c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why) return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "") @@ -408,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) { kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module, Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID, At: c.now()} - if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil { + if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil { return "", fmt.Errorf("the value was not kept: %v", err) } unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved) @@ -471,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string { said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...) continue } - vk := kv.level().ValueKey(kv.Key) + vk := keptLevel(kv).ValueKey(kv.Key) v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull + ", merged)", Approved: kv.Approved, At: c.now()} if err := c.Store.Put(ValuesState, vk, v); err != nil { diff --git a/modules/mesh-config/cmd/mesh-config/reads-outside b/modules/mesh-config/cmd/mesh-config/reads-outside new file mode 100644 index 00000000..ca488b77 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-config/module.json diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index 761706d8..e32b4dda 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -12,10 +12,19 @@ import ( // Reading the sources, writing the resolved values, and publishing the list of predefined keys. -// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the -// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the -// controller (issue 1953, ADR 0565 §3). +// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names +// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations +// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror. func (c *Config) sources() (Sources, error) { + sw, err := c.source() + if err != nil { + return Sources{}, err + } + return c.sourcesFrom(sw.Source) +} + +// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror. +func (c *Config) sourcesFrom(source string) (Sources, error) { s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{}, Predefined: map[string]Published{}, Values: map[string]Value{}} var problems []string @@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) { if err := c.readDeclarations(&s); err != nil { return s, fmt.Errorf("the modules' declarations cannot be read: %w", err) } + if source == SourceController { + c.mirrorDeclarations(&s) + } for _, u := range s.Unresolved { problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why)) } @@ -350,6 +362,10 @@ type Resolution struct { Moved []Moved `json:"moved"` Published string `json:"published,omitempty"` Withheld string `json:"withheld,omitempty"` + // Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates + // did after the take-over. + Declared []string `json:"declared,omitempty"` + Candidates []string `json:"candidates,omitempty"` } // Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources @@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) { c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " + "keys changed", At: c.now(), Moved: moved}) } + if written, derr := c.writeDeclared(s); derr != nil { + c.logf("[mesh-config] declared is not all written: %v", derr) + } else if len(written) > 0 { + r.Declared = written + } + if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer { + r.Candidates = c.takeCandidates() + } return r, err } -// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published -// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the +// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the +// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins; +// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the // list published stays, and why is said in every listing and in the log, because a value for a key no longer on // the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused). // It answers what it published, empty when nothing changed. func (c *Config) publish() (string, error) { - e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + sw, err := c.source() if err != nil { - return "", fmt.Errorf("the controller's list cannot be read: %w", err) + return "", err } - if e == nil { + read, whose := c.mirrorList, "the controller's list" + if sw.Source == SourceIndexer { + read, whose = c.indexerList, "the indexer's list" + } + list, found, err := read() + if err != nil { + return "", fmt.Errorf("%s cannot be read: %w", whose, err) + } + if !found { return "", nil // no list kept yet } - var list PredefinedList - if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { - return "", fmt.Errorf("the controller's list is not one: %s", e.Value) - } current := map[string]Published{} if err := c.each(PredefinedState, func(key string, raw []byte) error { var p Published @@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) { }); err != nil { return "", err } - generation := 0 + generation, sameCommit := 0, len(current) > 0 for _, p := range current { generation = max(generation, p.Generation) + sameCommit = sameCommit && p.Commit == list.Commit } - if len(current) > 0 && generation >= list.Generation { + if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) { c.withheld = "" return "", nil } diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go new file mode 100644 index 00000000..7fbc2ffc --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -0,0 +1,79 @@ +package main + +// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs +// and as the module's own tools beside them, the read of `declared` and resync. + +import ( + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The switch's tools. +const ( + CompareTool = "config_compare" + TakeOverTool = "config_take_over" + HandBackTool = "config_hand_back" + DeclaredTool = "config_declared" + ResyncTool = "config_resync" +) + +// SeatVerbs are the seat's verbs, as the manifest claims them. +var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb} + +func switchTools(c *Config) []stdio.Tool { + own := ownSwitchTools(c) + byName := map[string]stdio.Tool{} + for _, t := range own { + byName[t.Name] = t + } + for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} { + t := byName[tool] + t.Name = Seat + "." + verb + own = append(own, t) + } + return own +} + +func ownSwitchTools(c *Config) []stdio.Tool { + why := map[string]any{"type": "object", "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}} + whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: CompareTool, + Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " + + "each placed module's declarations (declared against the mirror declared-config), the list of predefined " + + "keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " + + "against the controller's pull-declares), and every placed module's value on every node resolved both " + + "ways; each with its reason. The take-over is refused while any stands.", + Run: func(map[string]any) (any, error) { return c.Compare() }}, + {Name: TakeOverTool, + Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " + + "the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " + + "Refused while compare shows a difference; on Approve switched if nothing differs then.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: HandBackTool, + Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " + + "pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: DeclaredTool, + Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " + + "commit they come from and how many keys, or one module whole; and which source holds now.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module, whole"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + return c.Declared(strings.TrimSpace(m)) + }}, + {Name: ResyncTool, + Description: "Read every source again and write what differs: the published list of predefined keys, the " + + "resolved values, declared, and after the take-over the indexer's config candidates.", + Run: func(map[string]any) (any, error) { return c.Resync() }}, + } +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover.go b/modules/mesh-config/cmd/mesh-config/takeover.go new file mode 100644 index 00000000..dad88461 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover.go @@ -0,0 +1,771 @@ +package main + +// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of +// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every +// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list, +// each open pull request's config declarations, and every placed module's resolved value on every node both ways; +// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the +// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write. +// +// **Which source holds** is the key `switch` alone, read by mesh-config alone: +// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list +// of predefined keys from the controller's mirror, a pull request's declarations from the controller's +// pull-declares; +// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from +// the indexer's `config-candidates`; pull-declares refused; +// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror, +// pull-declares taken. + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The states and reads of the switch. +const ( + // DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3). + DeclaredState = "declared" + // TakeoverState holds the key SwitchKey and each take-over ask, `ask.`. + TakeoverState = "takeover" + SwitchKey = "switch" + + // MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module: + // what `compare` holds the own source to, and the source after a hand-back. + MirrorDeclaredRead = "mesh-controller.declared-config" + // IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2). + IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState + CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState + // LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13). + LeaseRead = buscompose.ControllerLease +) + +// The sources the switch names. +const ( + SourceInterim = "interim" + SourceIndexer = "indexer" + SourceController = "controller" +) + +// TakeOverVerb and HandBackVerb are the seat's verbs. +const ( + TakeOverVerb = "take-over" + HandBackVerb = "hand-back" + CompareVerb = "compare" +) + +// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask. +type Switch struct { + Source string `json:"source"` + At time.Time `json:"at"` + By string `json:"by"` + Why string `json:"why"` + Ask string `json:"ask,omitempty"` +} + +// source is the source that holds now. +func (c *Config) source() (Switch, error) { + e, err := c.Store.Get(TakeoverState, SwitchKey) + if err != nil { + return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err) + } + if e == nil { + return Switch{Source: SourceInterim}, nil + } + var s Switch + if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) { + return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value) + } + return s, nil +} + +// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading). +func (c *Config) leaseAway() bool { + e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +} + +// ---- declared -------------------------------------------------------------------------------------------- + +// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.` +// names holds them, with that manifest's digest and commit (ADR 0565 §3). +type DeclaredKept struct { + Module string `json:"module"` + Manifest string `json:"manifest"` + Commit string `json:"commit,omitempty"` + Config map[string]Declaration `json:"settings"` +} + +// trunkKept is a module's declarations at the trunk with the manifest they come from. +func (c *Config) trunkKept(module string) (DeclaredKept, error) { + key, err := delivery.ModulesKey(module) + if err != nil { + return DeclaredKept{}, err + } + e, err := c.Store.Get(ModulesRead, key) + if err != nil || e == nil { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err) + } + var m struct { + Manifest string `json:"manifest"` + Commit string `json:"commit"` + } + if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module) + } + d, err := c.manifestDeclarations(module, m.Manifest) + if err != nil { + return DeclaredKept{}, err + } + return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil +} + +// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs +// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written. +func (c *Config) writeDeclared(s Sources) ([]string, error) { + held := map[string][]byte{} + if err := c.each(DeclaredState, func(key string, raw []byte) error { + held[key] = raw + return nil + }); err != nil { + return nil, err + } + var written []string + var problems []error + placed := map[string]bool{} + for _, module := range modulesOf(s.Assignments) { + placed[module] = true + k, err := c.trunkKept(module) + if err != nil { + continue // unresolved, said by the resolution; its key stays as it was + } + raw, _ := json.Marshal(k) + if bytes.Equal(raw, held[module]) { + continue + } + if err := c.Store.Put(DeclaredState, module, k); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + continue + } + written = append(written, module) + } + for module := range held { + if !placed[module] { + if err := c.Store.Delete(DeclaredState, module); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + } + } + } + return written, errors.Join(problems...) +} + +// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source +// after a hand-back, and the other side of compare. +func (c *Config) mirrorDeclarations(s *Sources) { + s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil + for _, module := range modulesOf(s.Assignments) { + e, err := c.Store.Get(MirrorDeclaredRead, module) + var d ModuleDeclarations + if err == nil && e != nil { + d, err = ParseModuleDeclarations(e.Value) + } else if err == nil { + err = errors.New("the mirror holds no key for it") + } + if err != nil { + for _, a := range s.Assignments { + if a.Module == module { + s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module, + Why: "the controller's mirror declared-config: " + err.Error()}) + } + } + continue + } + s.Declared[module] = d + } +} + +// ---- the predefined list and the pull requests from the indexer ------------------------------------------ + +// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit. +func (c *Config) indexerList() (PredefinedList, bool, error) { + e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var kept delivery.PredefinedKept + if err := json.Unmarshal(e.Value, &kept); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err) + } + list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit} + if err := json.Unmarshal(kept.List, &list.Keys); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err) + } + return list, true, nil +} + +// mirrorList is the controller's mirror's list. +func (c *Config) mirrorList() (PredefinedList, bool, error) { + e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var list PredefinedList + if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { + return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value) + } + return list, true, nil +} + +// candidates is every open pull request's config declarations as the indexer keeps them, by pull key. +func (c *Config) candidates() (map[string]PullDecl, error) { + out := map[string]PullDecl{} + err := c.each(CandidatesRead, func(key string, raw []byte) error { + var cand delivery.ConfigCandidate + if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" { + return fmt.Errorf("the indexer's config candidate %s is not one", key) + } + p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At} + for module, keys := range cand.Declares { + raw, _ := json.Marshal(keys) + var d map[string]Declaration + if err := json.Unmarshal(raw, &d); err != nil { + return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module) + } + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } + d[k] = decl + } + p.Declares[module] = d + } + out[pullKey(cand.Pull)] = p + return nil + }) + return out, err +} + +// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one +// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502 +// §6). The caller holds the lock. +func (c *Config) takeCandidates() []string { + cands, err := c.candidates() + if err != nil { + return []string{"the indexer's config candidates cannot be read: " + err.Error()} + } + var s Sources + if err := c.readPulls(&s); err != nil { + return []string{err.Error()} + } + var said []string + for _, k := range sortedKeys(cands) { + p := cands[k] + if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) { + continue + } + a, err := c.declarePull(p.Pull, p.Head, p.Declares) + if err != nil { + said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err)) + continue + } + said = append(said, a.Said+" (from the indexer)") + } + return said +} + +// declaresDigest is what a pull request's declarations hold a value to, in one encoding. +func declaresDigest(d map[string]map[string]Declaration) string { + flat := map[string]string{} + for m, keys := range d { + for k, decl := range keys { + flat[m+"."+k] = declDigest(decl) + } + } + raw, _ := json.Marshal(flat) + return string(raw) +} + +// ---- compare ---------------------------------------------------------------------------------------------- + +// Difference is one thing the own sources and the controller's mirrors say differently, and why. +type Difference struct { + What string `json:"what"` + Module string `json:"module,omitempty"` + Node string `json:"node,omitempty"` + Key string `json:"key,omitempty"` + Pull string `json:"pull,omitempty"` + Own json.RawMessage `json:"own,omitempty"` + Mirror json.RawMessage `json:"mirror,omitempty"` + Why string `json:"why"` +} + +// The kinds of difference. +const ( + DiffDeclared = "declared" + DiffPredefined = "predefined" + DiffPull = "pull" + DiffValue = "value" +) + +// Comparison is what compare answers. +type Comparison struct { + Source string `json:"source"` + Differences []Difference `json:"differences"` + Count int `json:"count"` + Said string `json:"said"` +} + +// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock. +func (c *Config) Compare() (*Comparison, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.compare() +} + +// compare is Compare; the caller holds the lock. +func (c *Config) compare() (*Comparison, error) { + sw, err := c.source() + if err != nil { + return nil, err + } + own, err := c.sourcesFrom(SourceInterim) + if err != nil { + return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err) + } + mirror := own + mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil + c.mirrorDeclarations(&mirror) + var diffs []Difference + + // Each placed module's declarations: `declared` against the mirror's key for it. + for _, module := range modulesOf(own.Assignments) { + var ownRaw []byte + if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil { + var k DeclaredKept + if json.Unmarshal(e.Value, &k) == nil { + ownRaw, _ = json.Marshal(normal(k.Config)) + } + } + var mirrorRaw []byte + if d, ok := mirror.Declared[module]; ok { + mirrorRaw, _ = json.Marshal(normal(d.Config)) + } + switch { + case ownRaw == nil && mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"}) + case ownRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw, + Why: "the mirror holds its declarations and declared does not"}) + case mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, + Why: "declared holds its declarations and the mirror does not"}) + case !bytes.Equal(ownRaw, mirrorRaw): + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw, + Why: "the trunk's manifest declares it otherwise than the mirror"}) + } + } + + // The predefined list: the indexer's against the mirror's. + il, iok, ierr := c.indexerList() + ml, mok, merr := c.mirrorList() + switch { + case ierr != nil || merr != nil: + diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()}) + case !iok || !mok: + if iok != mok { + diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+ + "mirror holds one: %v", iok, mok)}) + } + default: + for _, k := range unionKeys(il.Keys, ml.Keys) { + a, aok := il.Keys[k] + b, bok := ml.Keys[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"}) + case !bytes.Equal(ar, br): + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br, + Why: "the two lists declare it otherwise"}) + } + } + } + + // Each open pull request: the indexer's candidate against what the controller told by pull-declares. + cands, cerr := c.candidates() + if cerr != nil { + diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()}) + } + for _, k := range unionKeys(cands, own.Pulls) { + a, aok := cands[k] + b, bok := own.Pulls[k] + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"}) + case a.Head != b.Head: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+ + "the controller told %s", short(a.Head), short(b.Head))}) + case declaresDigest(a.Declares) != declaresDigest(b.Declares): + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"}) + } + } + + // Every placed module's value on every node, resolved both ways. + ownValues, mirrorValues := Resolve(own), Resolve(mirror) + for _, k := range unionKeys(ownValues, mirrorValues) { + a, aok := ownValues[k] + b, bok := mirrorValues[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + node, module, key := a.Node, a.Module, a.Key + if !aok { + node, module, key = b.Node, b.Module, b.Key + } + d := Difference{What: DiffValue, Node: node, Module: module, Key: key} + switch { + case !aok: + d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node" + case !bok: + d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it" + case !bytes.Equal(ar, br): + d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source" + default: + continue + } + diffs = append(diffs, d) + } + for _, u := range mirror.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why}) + } + for _, u := range own.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why}) + } + sort.SliceStable(diffs, func(i, j int) bool { + a, b := diffs[i], diffs[j] + return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull + }) + if diffs == nil { + diffs = []Difference{} + } + cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)} + if len(diffs) == 0 { + cmp.Said = "nothing differs: the take-over may be asked" + } else { + cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs)) + } + return cmp, nil +} + +// normal is declarations in one encoding: a null default as none. +func normal(d map[string]Declaration) map[string]Declaration { + out := map[string]Declaration{} + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } else { + decl.Default = compact(decl.Default) + } + out[k] = decl + } + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// ---- take-over and hand-back --------------------------------------------------------------------------------- + +// TakeoverAsk is one take-over asked of the operator, `takeover.ask.`: what the ask is composed from again, and +// where it stands. +type TakeoverAsk struct { + ID string `json:"id"` + From string `json:"from"` + Why string `json:"why"` + At time.Time `json:"at"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Decided time.Time `json:"decided,omitzero"` + Answered string `json:"answered,omitempty"` + Recorded bool `json:"recorded,omitempty"` +} + +func askKey(id string) string { return "ask." + id } + +// act is what an answer to the take-over binds. +func (t TakeoverAsk) act(option string) asks.Act { + return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From, + "at": t.At.UTC().Format(time.RFC3339Nano)} +} + +// ask is the take-over's ask, composed from it alone. +func (t TakeoverAsk) ask() asks.Ask { + why, _ := sayable("Why: " + t.Why) + a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources", + Explanation: strings.Join([]string{ + "mesh-config would read the list of predefined keys and each pull request's config from the delivery's " + + "indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.", + point(why)}, "\n"), + Who: asks.Operator, Expires: t.At.Add(ProposalLasts), + OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover", + Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " + + "to the controller's records until the controller's lease is handed away."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"}, + {OptionDecline, "Decline", "nothing changes"}, + } { + binds, _ := asks.ActDigest(t.act(o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// TakeoverAnswer is what take-over and hand-back answer. +type TakeoverAnswer struct { + Said string `json:"said"` + Ask string `json:"ask,omitempty"` + Source string `json:"source"` +} + +// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference. +func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why the take-over is asked now: it is shown to the operator") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + if c.Asks == nil { + return nil, errors.New("the operator cannot be asked from here, so nothing is switched") + } + c.mu.Lock() + defer c.mu.Unlock() + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceIndexer { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " + + sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil + } + cmp, err := c.compare() + if err != nil { + return nil, err + } + if cmp.Count > 0 { + first := cmp.Differences[0] + return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+ + "Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why) + } + now := c.now() + var open []string + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting && + now.Before(t.At.Add(ProposalLasts)) { + open = append(open, t.ID) + } + return nil + }) + if len(open) > 0 { + return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0]) + } + t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(), + State: ProposalWaiting} + ask := t.ask() + if err := ask.Check(now); err != nil { + return nil, err + } + if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil { + return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err) + } + if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil { + t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error() + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return nil, fmt.Errorf("the operator could not be asked: %w", err) + } + return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+ + "mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+ + "then", ask.Headline)}, nil +} + +// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock. +func (c *Config) decideTakeovers() []string { + var said []string + var waiting []TakeoverAsk + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting { + waiting = append(waiting, t) + } + return nil + }) + now := c.now() + for _, t := range waiting { + keep := func(state, outcome string) string { + t.State, t.Outcome, t.Decided = state, outcome, now + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return "take-over " + t.ID + ": " + outcome + } + e, err := c.Asks.Record(t.ID) + if err != nil || e == nil { + if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) { + said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched")) + } + continue + } + var rec routerRecord + if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != t.ID { + said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched")) + continue + } + if w.Outcome != asks.OutcomeChosen { + said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched")) + continue + } + option, err := w.For(Module, t.ask()) + if err == nil { + err = option.Performs(t.act(option.ID)) + } + if err != nil { + said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error())) + continue + } + t.Answered = w.Says() + if option.ID != OptionApprove { + said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched")) + continue + } + cmp, err := c.compare() + switch { + case err != nil: + said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error())) + continue + case cmp.Count > 0: + said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+ + "now; nothing was switched, ask again once it is empty", cmp.Count))) + continue + } + sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID} + if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil { + said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+ + "on the next look", t.ID, err)) + continue + } + if err := c.Asks.Warranted(t.ID); err == nil { + t.Recorded = true + } + said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now")) + } + return said +} + +func approvedTakeover(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) +} + +// HandBack returns to the controller's records until the controller's lease is handed away. +func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why config hands back: it is kept with the switch") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + c.mu.Lock() + defer c.mu.Unlock() + if c.leaseAway() { + return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " + + "(novox/hq ADR 0565 §4)") + } + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceController { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil + } + now := c.now() + back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)} + if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil { + return nil, fmt.Errorf("the switch could not be written: %w", err) + } + answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " + + "declared-config and predefined, and pull-declares"} + if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil { + answer.Said += "; the hand-act log does not hold it: " + err.Error() + } + return answer, nil +} + +// Declared is the read of `declared`: every module's key, or one whole. +func (c *Config) Declared(module string) (any, error) { + if module != "" { + e, err := c.Store.Get(DeclaredState, module) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module) + } + return json.RawMessage(e.Value), nil + } + rows := map[string]any{} + err := c.each(DeclaredState, func(key string, raw []byte) error { + var k DeclaredKept + if json.Unmarshal(raw, &k) == nil { + rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)} + } + return nil + }) + sw, _ := c.source() + return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err +} + +// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and +// after the take-over the indexer's config candidates (ADR 0565 §15). +func (c *Config) Resync() (any, error) { + r, err := c.Reconcile() + if err != nil { + return r, err + } + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]any{"resolution": r} + if sw, err := c.source(); err == nil && sw.Source == SourceIndexer { + out["candidates"] = c.takeCandidates() + } + return out, nil +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover_test.go b/modules/mesh-config/cmd/mesh-config/takeover_test.go new file mode 100644 index 00000000..c66b21f8 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations. +const ( + claudeDeclared = `{"module":"claude-code","settings":{ + "managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}` + oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}` +) + +// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's +// declarations of every placed module, and the indexer's predefined list equal to the mirror's. +func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) { + t.Helper() + c, m, r := aProposingMesh(t) + m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared) + m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared) + m.mirror(t, MirrorDeclaredRead, "old", oldDeclared) + indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + return c, m, r +} + +// indexerList is the indexer's `predefined.list` at a commit. +func indexerList(t *testing.T, m *memStore, commit, list string) { + t.Helper() + raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)}) + m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw)) +} + +// candidate is the indexer's config candidate of a pull request. +func candidate(t *testing.T, m *memStore, pull, head, declares string) { + t.Helper() + key, err := delivery.ConfigCandidateKey(pull) + if err != nil { + t.Fatal(err) + } + m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`) +} + +func kinds(cmp *Comparison) []string { + var out []string + for _, d := range cmp.Differences { + k := d.What + ":" + d.Module + d.Pull + d.Key + if d.Node != "" { + k += "@" + d.Node + } + if !slices.Contains(out, k) { + out = append(out, k) + } + } + return out +} + +// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's +// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and +// its insight of 2026-10-29; issue 1953 L5-C4). +func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) { + cases := []struct { + name string + change func(t *testing.T, m *memStore) + want []string + }{ + {"all agree", func(*testing.T, *memStore) {}, nil}, + {"the mirror declares a default otherwise", func(t *testing.T, m *memStore) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + }, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}}, + {"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") }, + []string{"declared:old", "value:old@g14", "value:oldworkers@g14"}}, + {"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) { + m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1)) + }, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558) + {"the lists differ", func(t *testing.T, m *memStore) { + indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + }, []string{"predefined:agent-managed-settings"}}, + {"a pull request only the indexer holds", func(t *testing.T, m *memStore) { + candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + }, []string{"pull:novox/mesh-catalog#9"}}, + } + for _, tc := range cases { + c, m, r := anAgreeingMesh(t) + tc.change(t, m) + cmp, err := c.Compare() + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if got := kinds(cmp); !slices.Equal(got, tc.want) { + t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences) + } + for _, d := range cmp.Differences { + if d.Why == "" { + t.Errorf("%s: a difference without its reason: %+v", tc.name, d) + } + } + a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"}) + if len(tc.want) > 0 { + if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 { + t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err) + } + continue + } + if err != nil || a.Ask == "" || len(r.subjects) != 1 { + t.Errorf("%s: the take-over was not asked: %v", tc.name, err) + } + } +} + +// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests' +// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the +// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13). +func TestTheTakeOverAndTheHandBack(t *testing.T) { + c, m, r := anAgreeingMesh(t) + steps := []struct { + name string + do func() (string, error) + want string // what the answer or the error says + }{ + {"declared is written when it differs, and once", func() (string, error) { + _ = m.Delete(DeclaredState, "dunst") + first, err := c.Reconcile() + again, _ := c.Reconcile() + return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err + }, "dunst|"}, + {"pull-declares taken before the take-over", func() (string, error) { + a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0", + Declares: map[string]any{}}) + return answered(a), err + }, "keeps what"}, + {"the take-over is asked", func() (string, error) { + _ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate + a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"}) + return answered(a), err + }, "is asked"}, + {"the operator approves: switched to the indexer", func() (string, error) { + r.choose(t, r.lastAsk(t), OptionApprove, nil) + out, err := c.Decide() + return strings.Join(out, "; "), err + }, "reads its own sources from now"}, + {"pull-declares refused after it, naming the indexer", func() (string, error) { + _, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1", + Declares: map[string]any{}}) + return "", err + }, "config-candidates"}, + {"the list comes from the indexer", func() (string, error) { + indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72}, + "agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "theme":{"type":"string","description":"the desktop's theme"}}`) + res, err := c.Reconcile() + return res.Published, err + }, "theme"}, + {"a pull request's config comes from the indexer", func() (string, error) { + candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + res, err := c.Reconcile() + return strings.Join(res.Candidates, "; "), err + }, "from the indexer"}, + {"hand-back returns to the mirrors", func() (string, error) { + a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"}) + return answered(a), err + }, "controller's records again"}, + {"after it, declarations resolve from the mirror", func() (string, error) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + _, err := c.Reconcile() + return string(m.resolved(t, "g14", "dunst", "width").Value), err + }, "400"}, + {"hand-back is gone after the lease write", func() (string, error) { + m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`) + _, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"}) + return "", err + }, "gone"}, + {"the terminal sets no trusted value after the lease write", func() (string, error) { + _, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{}, + Why: "x", Caller: ControllerCaller, By: "person.jochen"}) + return "", err + }, "propose"}, + } + for _, s := range steps { + got, err := s.do() + text := got + if err != nil { + text = err.Error() + } + if !strings.Contains(text, s.want) { + t.Errorf("%s: said %q, want %q", s.name, text, s.want) + } + } + if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") { + t.Errorf("the hand-back is not recorded: %v", m.handActs) + } + if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) { + t.Errorf("declared holds no dunst with its manifest: %v", e) + } + // An Approve that finds compare differing at the answer switches nothing. + c2, m2, r2 := anAgreeingMesh(t) + if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil { + t.Fatal(err) + } + m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + r2.choose(t, r2.lastAsk(t), OptionApprove, nil) + out, _ := c2.Decide() + if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") { + t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out) + } +} + +// lastAsk is the newest ask the router took. +func (r *standIn) lastAsk(t *testing.T) string { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + var newest string + var at time.Time + for id, a := range r.asks { + if newest == "" || a.Expires.After(at) { + newest, at = id, a.Expires + } + } + if newest == "" { + t.Fatal("the router took no ask") + } + return newest +} + +func answered(a any) string { + raw, _ := json.Marshal(a) + return string(raw) +} diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 6a0791d7..e6d2dae0 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,4 +2,13 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 8c9ba7ca..b8d81e43 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,2 +1,26 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 66db5b60..9554db29 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -7,7 +7,10 @@ "name": "mesh-config", "scope": "mesh", "serves": [ - "config" + "config", + "compare", + "take-over", + "hand-back" ] } ], @@ -23,6 +26,8 @@ "gitea.forge.pull-request.closed" ], "invokes": [ + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", "seat:mesh-controller.hand-act", "seat:mesh-controller.warranted", "gitea.gitea_get_pull_request" @@ -33,7 +38,18 @@ "mesh-delivery-builder.builds", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.manifests", - "mesh-controller.predefined" + "mesh-controller.predefined", + "mesh-controller.declared-config", + "mesh-delivery-indexer.predefined", + "mesh-delivery-indexer.config-candidates", + "mesh-controller.lease" + ], + "tools": [ + "config_compare", + "config_take_over", + "config_hand_back", + "config_declared", + "config_resync" ], "state": [ "values", @@ -41,7 +57,9 @@ "predefined", "proposals", "pulls", - "kept" + "kept", + "declared", + "takeover" ], "logs": [ "history" diff --git a/modules/mesh-hand-acts/.gitignore b/modules/mesh-hand-acts/.gitignore new file mode 100644 index 00000000..2a60d0a0 --- /dev/null +++ b/modules/mesh-hand-acts/.gitignore @@ -0,0 +1,2 @@ +/mesh-hand-acts +/cmd/mesh-hand-acts/mesh-hand-acts diff --git a/modules/mesh-hand-acts/README.md b/modules/mesh-hand-acts/README.md new file mode 100644 index 00000000..b435e08d --- /dev/null +++ b/modules/mesh-hand-acts/README.md @@ -0,0 +1,26 @@ +# mesh-hand-acts + +Holds the seat `mesh-hand-acts` (novox/hq ADR 0565 §10, §11): the hand-act log, moved out of the controller. Go, one +binary (`cmd/mesh-hand-acts`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes the log. + +## What it keeps + +- the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream): + every act appended in its order under its key `act--` (`warrant--` for a warrant), in the layout + of the controller's log. The log keeps each act 400 days (`max-age-days`) and refuses a new act when + full rather than drop one; `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`. +- `mesh-hand-acts_warrants`: one key `.` per warrant, made once by create before its act is appended, so of + several instances one records it; a warrant whose act was lost is written again from the router's record. + +## Its verbs + +- `mesh-hand-acts.hand-act`: what, why, cause, the condition addressed, and the act's verb (a repair when not given; + `hand-act drill` for a drill, whose cause is always `drill`). Who did it is the caller the runtime proves. +- `mesh-hand-acts.warranted`: asker and ask. Read from the router's own record of that ask (`messenger.asks`), never + from the caller, and recorded once per ask; an ask still open, ended without a choice or another asker's is refused. +- `mesh-hand-acts.hand-acts`: both logs, this one and the controller's `mesh-controller_hand-acts` until its lease + ends, newest first, with the causes repaired by hand twice within a fortnight; `warrants=true` lists the warrants. + +No act is written to both logs: every recorder calls this seat from its delivery, and the controller keeps its own +acts in its own bucket until it loses its lease. diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go new file mode 100644 index 00000000..6e8579e8 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go @@ -0,0 +1,375 @@ +package main + +// The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a +// person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat +// mesh-hand-acts, once for the mesh, and is the one writer of its log `acts` — a stream, as ADR 0565 §10 has it: +// the mesh's module log (ADR 0297), append-only, every act kept in its order under its key, refusing a new act when +// full rather than dropping an old one, so every act is held 400 days and longer — and of its bucket `warrants` (one +// key `.` made once by create). Each act is said as `act-recorded`. +// +// **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the +// router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who +// chose, through which channel, with which proofs and which answer, once per ask. The controller's bucket +// `mesh-controller_hand-acts` stays its own until it loses its lease and is read, never written: `hand-acts` answers +// over both logs, and no act is written to both (ADR 0565 §11). + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// The seat, the log, the states and the event, as mesh-sdk's handacts names them. +const ( + Module = handacts.Module + Seat = handacts.Seat + ActsLog = handacts.ActsLog + WarrantsState = handacts.WarrantsState + ControllerActs = handacts.ControllerActs + // RouterRecords is the router's record of every ask, `.` (ADR 0259): the warrant is read there. + RouterRecords = "messenger.asks" + RecordedEvent = handacts.EventRecorded + // RecordedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + RecordedSubject = "mesh.seat." + Seat + ".event." + RecordedEvent + + VerbRecord = handacts.VerbRecord + VerbDrill = handacts.VerbDrill + VerbWarrant = handacts.VerbWarrant + CauseDrill = handacts.CauseDrill + CauseOperatorAnswer = handacts.CauseOperatorAnswer +) + +// RepeatedWithin is how far back a cause counts as repeated (S15). +const RepeatedWithin = 14 * 24 * time.Hour + +// Act is one entry of the log: mesh-sdk's handacts.Act, the controller's HandAct unchanged. +type Act = handacts.Act + +// Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only +// when it has none, and says ErrExists otherwise. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Create(state, key string, value any) error + Keys(state string) ([]string, error) + // Append adds an entry to the end of a log; Last is a key's newest entry; Entries every entry in order; Fill how + // many entries and bytes the log holds against its cap. + Append(log, key string, value any) error + Last(log, key string) ([]byte, bool, error) + Entries(log string) ([][]byte, error) + Fill(log string) (Fill, error) +} + +// Fill is how full a log is: its entries, its bytes and its cap in bytes. +type Fill struct { + Entries uint64 `json:"entries"` + Bytes uint64 `json:"bytes"` + MaxBytes uint64 `json:"max-bytes"` +} + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Log is mesh-hand-acts: the log and what records into it. +type Log struct { + Store Store + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) +} + +var seq atomic.Uint64 + +// actKey is an act's key: its time and a sequence, so the keys list in order, as the controller's do. +func actKey(at time.Time) string { + return "act-" + strconv.FormatInt(at.UnixNano(), 10) + "-" + strconv.FormatUint(seq.Add(1), 10) +} + +// Recorded is what a recorder is answered: the act as kept. +type Recorded struct { + Act Act `json:"act"` + Said string `json:"said"` +} + +// HandAct records one act a caller says it did, by hand or on purpose: what, why, its cause and the condition it +// addresses. by is the caller the runtime proved. verb is the act's verb as the log names it: a repair by default, a +// drill with VerbDrill, or a recorder's own (`conditions silence`, `config set`), which S15 reads as it reads the +// controller's. +func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, error) { + what, why, cause, verb = strings.TrimSpace(what), strings.TrimSpace(why), strings.TrimSpace(cause), strings.TrimSpace(verb) + switch { + case !proven(by): + return Recorded{}, fmt.Errorf("who acts is the caller the runtime proves, and it proved none (%q): nothing was recorded", by) + case what == "": + return Recorded{}, errors.New("a hand act says what was done: nothing was recorded") + case why == "": + return Recorded{}, errors.New("a hand act says why: nothing was recorded") + } + if verb == "" { + verb = VerbRecord + } + if verb == VerbDrill { + cause = CauseDrill + } + switch { + case cause == "": + return Recorded{}, errors.New("a hand act says its cause, in a word or a condition's kind: the word a second act " + + "for the same reason uses, which is how a repair done twice is found. Nothing was recorded") + case strings.EqualFold(cause, CauseDrill) && verb != VerbDrill: + return Recorded{}, errors.New("a drill is not recorded as a repair: record it with verb " + VerbDrill + + ", which no healer is wanted for. Nothing was recorded") + } + at := l.Now().UTC() + act := Act{ID: actKey(at), At: at, By: by, Verb: verb, Args: []string{what}, Why: why, Cause: cause, + Condition: strings.TrimSpace(condition)} + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s did %q, because %q (cause: %s)", act.ID, by, what, + why, cause)}, nil +} + +// record writes one act and says it. An act not said is still kept, and the log says so. +func (l *Log) record(act Act) error { + if err := l.Store.Append(ActsLog, act.ID, act); err != nil { + return fmt.Errorf("the act could not be recorded: %w", err) + } + if l.Publish != nil { + if err := l.Publish(RecordedSubject, act, act.ID); err != nil { + l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err) + } + } + return nil +} + +// routerRecord is what warranted reads of the router's record of an ask. +type routerRecord struct { + State string `json:"state"` + Warrant *asks.Warrant `json:"warrant"` + Ask map[string]any `json:"ask"` +} + +var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// Warrant is one ask's warrant as recorded: the key `.` of `warrants`, naming the act it recorded. +type Warrant struct { + Act string `json:"act"` + At time.Time `json:"at"` + By string `json:"recorded-at-the-word-of"` +} + +// Warranted records an asker's act on the operator's warrant for its ask, read from the router's record, once per ask +// however often it is asked (ADR 0274, ADR 0259). The caller gives which ask and nothing else. +func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) { + switch { + case !proven(caller): + return Recorded{}, fmt.Errorf("who asks is the caller the runtime proves, and it proved none (%q): nothing was recorded", caller) + case !askerModule.MatchString(asker): + return Recorded{}, fmt.Errorf("%q is not a module's name", asker) + case !asks.UsableID(ask): + return Recorded{}, fmt.Errorf("%q is not an ask's id", ask) + } + key := asker + "." + ask + claimed, found, err := l.Store.Get(WarrantsState, key) + if err != nil { + return Recorded{}, err + } + if found { + if r, ok, err := l.already(key, claimed); err != nil || ok { + return r, err + } + } + raw, found, err := l.Store.Get(RouterRecords, key) + if err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err) + } + var rec routerRecord + if found { + if err := json.Unmarshal(raw, &rec); err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s is not readable: %w", asker, ask, err) + } + } + w := rec.Warrant + switch { + case rec.State == "open" || rec.State == "held": + return Recorded{}, fmt.Errorf("%s's ask %s is still %s: nobody has answered it. Nothing was recorded", asker, ask, rec.State) + case !found || rec.State == "" || w == nil: + return Recorded{}, fmt.Errorf("the router holds no closed record of %s's ask %s. Nothing was recorded", asker, ask) + case w.Asker != asker || w.Ask != ask: + return Recorded{}, fmt.Errorf("the router's record is for %s's ask %s. Nothing was recorded", w.Asker, w.Ask) + case w.Outcome != asks.OutcomeChosen || w.By == nil: + return Recorded{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record. Nothing was "+ + "recorded", asker, ask, w.Outcome) + case w.AskDigest == "": + return Recorded{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest. Nothing was recorded", asker, ask) + } + at := l.Now().UTC() + act := Act{ID: "warrant-" + asker + "-" + ask, At: w.At.UTC(), Verb: VerbWarrant, + Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)}, + Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker), + By: fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity), + Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs, + RequestedBy: asker + ", recorded at the word of " + caller, + Outcome: "chosen; what " + asker + " did with it is in its own record"} + if claimed == nil { + // The claim first: of any number of instances asking at once, the one whose create lands records the act. + if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) { + return Recorded{Said: "another instance records " + act.ID + " now"}, nil + } else if err != nil { + return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err) + } + } + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil +} + +// already answers a warrant recorded before, when its act is in the log; false when the act was lost between the +// claim and the log, so it is read from the router again and written. +func (l *Log) already(key string, raw []byte) (Recorded, bool, error) { + var w Warrant + if err := json.Unmarshal(raw, &w); err != nil { + return Recorded{}, false, fmt.Errorf("the warrant %s is not readable: %w", key, err) + } + actRaw, found, err := l.Store.Last(ActsLog, w.Act) + if err != nil || !found { + return Recorded{}, false, err + } + var act Act + _ = json.Unmarshal(actRaw, &act) + return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, true, nil +} + +func via(w asks.Warrant) string { + if w.By == nil { + return w.Channel + } + v := w.Channel + " (" + w.By.Kind + ")" + if w.By.Verified != "" { + v += ", " + w.By.Verified + } + return v +} + +// proven is a bus user the runtime proved: never empty, never the SDK's unproven word. +func proven(by string) bool { + by = strings.TrimSpace(by) + return by != "" && !strings.EqualFold(by, "unproven") && !strings.HasPrefix(by, "(") +} + +// Listing is what hand-acts answers: every act of both logs within the days asked, newest first, narrowed by cause +// and who, and the causes repaired by hand twice or more within a fortnight. +type Listing struct { + Acts []Act `json:"acts"` + Repeated map[string]int `json:"repeated"` + Days int `json:"days"` + // Unread says which log could not be read, so what is missing is said and never read as nothing. + Unread map[string]string `json:"unread,omitempty"` + Kept map[string]int `json:"kept"` + // Fill is how full this seat's log is against its cap: full, it refuses the next act. + Fill *Fill `json:"fill,omitempty"` +} + +// HandActs reads both logs. +func (l *Log) HandActs(days int, cause, who string) Listing { + now := l.Now() + since := now.Add(-time.Duration(days) * 24 * time.Hour) + out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}} + var all []Act + for _, log := range []string{ActsLog, ControllerActs} { + acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin))) + if err != nil { + if out.Unread == nil { + out.Unread = map[string]string{} + } + out.Unread[log] = err.Error() + continue + } + out.Kept[log] = kept + all = append(all, acts...) + } + for _, a := range all { + if now.Sub(a.At) <= RepeatedWithin && a.Repair() { + out.Repeated[a.Cause]++ + } + if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) { + continue + } + out.Acts = append(out.Acts, a) + } + for c, n := range out.Repeated { + if n < 2 { + delete(out.Repeated, c) + } + } + sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) }) + if f, err := l.Store.Fill(ActsLog); err == nil { + out.Fill = &f + } + return out +} + +// read is every act of one log since a moment, and how many it keeps: this seat's log entry by entry, the +// controller's bucket by its keys, each of which names its time, so only those within are read whole. +func (l *Log) read(log string, since time.Time) ([]Act, int, error) { + var out []Act + keep := func(raw []byte, from string) { + var a Act + if json.Unmarshal(raw, &a) != nil || a.At.Before(since) { + return + } + a.Log = from + out = append(out, a) + } + if log == ActsLog { + entries, err := l.Store.Entries(ActsLog) + if err != nil { + return nil, 0, err + } + for _, raw := range entries { + keep(raw, handacts.LogSeat) + } + return out, len(entries), nil + } + keys, err := l.Store.Keys(log) + if err != nil { + return nil, 0, err + } + for _, k := range keys { + if at, ok := keyTime(k); ok && at.Before(since) { + continue + } + raw, found, err := l.Store.Get(log, k) + if err != nil { + return nil, len(keys), err + } + if found { + keep(raw, handacts.LogController) + } + } + return out, len(keys), nil +} + +// keyTime is the time an act's key names (`act--`), when it names one. +func keyTime(key string) (time.Time, bool) { + parts := strings.Split(key, "-") + if len(parts) != 3 || parts[0] != "act" { + return time.Time{}, false + } + ns, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(0, ns), true +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go new file mode 100644 index 00000000..73d42cef --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "sync" + "testing" + "time" +) + +type entry struct { + key string + raw []byte +} + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + logs map[string][]entry + keysErr map[string]error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, logs: map[string][]entry{}, keysErr: map[string]error{}} +} + +func (s *memStore) Append(log, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return err + } + s.logs[log] = append(s.logs[log], entry{key, raw}) + return nil +} + +func (s *memStore) Last(log, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i := len(s.logs[log]) - 1; i >= 0; i-- { + if s.logs[log][i].key == key { + return s.logs[log][i].raw, true, nil + } + } + return nil, false, nil +} + +func (s *memStore) Entries(log string) ([][]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return nil, err + } + var out [][]byte + for _, e := range s.logs[log] { + out = append(out, e.raw) + } + return out, nil +} + +func (s *memStore) Fill(log string) (Fill, error) { + s.mu.Lock() + defer s.mu.Unlock() + return Fill{Entries: uint64(len(s.logs[log])), MaxBytes: 1 << 30}, nil +} + +// logKeys are a log's keys in order. +func (s *memStore) logKeys(log string) []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for _, e := range s.logs[log] { + out = append(out, e.key) + } + return out +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func newLog() (*Log, *memStore, *[]Act, *time.Time) { + store := newStore() + var said []Act + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now }, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(Act); ok && subject == "mesh.seat.mesh-hand-acts.event.act-recorded" && id == a.ID { + said = append(said, a) + } + return nil + }} + return l, store, &said, &now +} + +// hand-act records what the proven caller says it did, once, and says it; it refuses what cannot be read back, a +// caller the runtime did not prove, and a drill recorded as a repair (ADR 0565 §10; issue 1953 L5-C1). +func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) { + cases := []struct { + name, by, verb, what, why, cause string + wantVerb, wantCause, err string + }{ + {"a repair", "ace.mesh-config", "", "set dns", "it was wrong", "config", VerbRecord, "config", ""}, + {"a recorder's own verb", "novox.mesh-conditions", "conditions silence", "silenced x", "known", "operator-answer", + "conditions silence", "operator-answer", ""}, + {"a drill takes its cause", "novox.mesh-healers", VerbDrill, "stopped records", "does it raise", "", VerbDrill, CauseDrill, ""}, + {"no proven caller", "unproven", "", "x", "y", "z", "", "", "proves"}, + {"by said empty", "", "", "x", "y", "z", "", "", "proves"}, + {"no why", "person.jochen", "", "x", "", "z", "", "", "says why"}, + {"no what", "person.jochen", "", "", "y", "z", "", "", "says what"}, + {"no cause", "person.jochen", "", "x", "y", "", "", "", "cause"}, + {"a drill as a repair", "person.jochen", "", "x", "y", "drill", "", "", "not recorded as a repair"}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing") + keys := store.logKeys(ActsLog) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || len(keys) != 1 || keys[0] != r.Act.ID || len(*said) != 1 { + t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said)) + } + var kept Act + raw, _, _ := store.Last(ActsLog, keys[0]) + _ = json.Unmarshal(raw, &kept) + if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" || + kept.Args[0] != tc.what || kept.Log != "" { + t.Errorf("%s: kept %+v", tc.name, kept) + } + } +} + +// warranted records the router's word once per ask, never the caller's; it refuses an ask still open, one ended +// without a choice, another asker's, and one the router holds no record of (ADR 0565 §10, ADR 0259). +func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) { + chosen := `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"chosen","label":"Approve", + "by":{"who":"operator","kind":"telegram","identity":"42","verified":"P2"},"channel":"telegram","proofs":["P1","P2"], + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}` + cases := []struct { + name, record, asker, ask, err string + // claimed is a warrant claimed by an instance that stopped before its act reached the log. + claimed bool + }{ + {"chosen", chosen, "claude-code", "a1", "", false}, + {"claimed, its act lost: written now", chosen, "claude-code", "a1", "", true}, + {"still open", `{"state":"open"}`, "claude-code", "a1", "still open", false}, + {"no record", "", "claude-code", "a1", "no closed record", false}, + {"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`, + "claude-code", "a1", "no person chose", false}, + {"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1", + "record is for mesh-config", false}, + {"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest", false}, + {"not a module", chosen, "Claude Code", "a1", "not a module", false}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + if tc.record != "" { + _ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record)) + } + if tc.claimed { + _ = store.Create(WarrantsState, "claude-code.a1", Warrant{Act: "warrant-claude-code-a1"}) + } + r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask) + if tc.err != "" { + if keys := store.logKeys(ActsLog); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || r.Act.Verb != VerbWarrant || r.Act.Cause != CauseOperatorAnswer || + !strings.Contains(r.Act.By, "operator, as telegram identity 42") || r.Act.Via != "telegram (telegram), P2" || + !strings.Contains(r.Act.RequestedBy, "ace.claude-code") || len(r.Act.Proofs) != 2 { + t.Fatalf("%s: %+v, %v", tc.name, r.Act, err) + } + // Asked again, by another instance: nothing more is recorded. + again, err := l.Warranted("g14.claude-code", "claude-code", "a1") + keys := store.logKeys(ActsLog) + if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 { + t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said)) + } + } +} + +// hand-acts answers over both logs, newest first, narrowed by days, cause and who, and counts each cause repaired by +// hand twice in a fortnight across both, never a person's decision; a log it cannot read is said, never read as empty. +func TestHandActsReadsBothLogs(t *testing.T) { + l, store, _, now := newLog() + put := func(log, verb, cause, by string, ago time.Duration) { + at := now.Add(-ago) + a := Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"} + if log == ActsLog { + _ = store.Append(log, actKey(at), a) + return + } + _ = store.Put(log, actKey(at), a) + } + put(ActsLog, VerbRecord, "relogin", "person.jochen", time.Hour) + put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour) + put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour) + put(ActsLog, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour) + put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour) + cases := []struct { + name string + days int + cause, who string + want int + }{ + {"a fortnight", 14, "", "", 6}, + {"a day", 1, "", "", 5}, + {"forty days", 40, "", "", 7}, + {"by cause", 14, "relogin", "", 2}, + {"by who", 14, "", "claude-code", 1}, + } + for _, tc := range cases { + got := l.HandActs(tc.days, tc.cause, tc.who) + if len(got.Acts) != tc.want || len(got.Repeated) != 1 || got.Repeated["relogin"] != 2 || got.Unread != nil { + t.Errorf("%s: %d acts, repeated %v, unread %v", tc.name, len(got.Acts), got.Repeated, got.Unread) + } + for i := 1; i < len(got.Acts); i++ { + if got.Acts[i].At.After(got.Acts[i-1].At) { + t.Errorf("%s: not newest first", tc.name) + } + } + } + if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 || got.Fill == nil || + got.Fill.Entries != 4 { + t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept) + } + store.keysErr[ControllerActs] = errors.New("the bus is away") + if got := l.HandActs(14, "", ""); got.Unread[ControllerActs] == "" || len(got.Acts) != 4 { + t.Errorf("an unreadable log: %+v", got.Unread) + } + w, err := l.Warrants() + if err != nil || w.(map[string]any)["count"] != 0 { + t.Errorf("warrants answered %v, %v", w, err) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go new file mode 100644 index 00000000..b103fa23 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go @@ -0,0 +1,84 @@ +// mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, its module +// log: a stream that keeps every act in order) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for +// `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Create(state, key string, value any) error { + _, err := stdio.State(state).Create(key, value) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeStore) Append(log, key string, value any) error { + _, err := stdio.Log(log).Append(key, value) + return err +} + +func (runtimeStore) Last(log, key string) ([]byte, bool, error) { + e, err := stdio.Log(log).Last(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Entries(log string) ([][]byte, error) { + var out [][]byte + for from := uint64(0); ; { + page, err := stdio.Log(log).Read(from, "", 1000) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, e.Value) + } + if page.Next == 0 { + return out, nil + } + from = page.Next + } +} + +func (runtimeStore) Fill(log string) (Fill, error) { + i, err := stdio.Log(log).Info() + if err != nil { + return Fill{}, err + } + return Fill{Entries: i.Entries, Bytes: i.Bytes, MaxBytes: i.MaxBytes}, nil +} + +func main() { + l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf, + Publish: stdio.SeatPublish} + if err := stdio.Serve("", Tools(l)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go new file mode 100644 index 00000000..cd19669a --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565 +// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, keeps its +// acts as its log (a stream, §10) and warrants as a bucket that never ages, so a warrant is recorded once. Every state +// and log it keeps and reads has a tool that reads it (ADR 0565 §15). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Runs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + Logs []struct { + Name string `json:"name"` + MaxAgeDays int `json:"max-age-days"` + } `json:"logs"` + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.Runs != Module { + t.Errorf("module %q runs as %q", m.Module, m.Runs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v", m.Claims) + } + // act-recorded is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { + t.Errorf("emits %v, reads %v", m.Emits, m.Reads) + } + readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, + RouterRecords: WarrantedTool} + tools := map[string]bool{} + for _, tool := range Tools(&Log{}) { + tools[tool.Name] = true + } + // The acts are a log kept 400 days (ADR 0565 §10). + if len(m.Logs) != 1 || m.Logs[0].Name != ActsLog || m.Logs[0].MaxAgeDays != 400 { + t.Errorf("logs %+v, want the acts as a log kept 400 days", m.Logs) + } + for _, raw := range m.State { + var name string + if json.Unmarshal(raw, &name) != nil || name != WarrantsState { + t.Errorf("the state %s is not the warrants, or ages", raw) + } + if !tools[readBy[name]] { + t.Errorf("the state %s has no tool that reads it", name) + } + } + if !tools[readBy[ActsLog]] { + t.Errorf("the log %s has no tool that reads it", ActsLog) + } + for _, r := range m.Reads { + if !tools[readBy[r]] { + t.Errorf("the read %s has no tool that answers it", r) + } + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside new file mode 100644 index 00000000..60d6785c --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-hand-acts/module.json diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go new file mode 100644 index 00000000..833db625 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go @@ -0,0 +1,110 @@ +package main + +// The seat's verbs (novox/hq ADR 0565 §10, §15): hand-act and warranted record, hand-acts reads both logs and, with +// warrants, every warrant recorded. A recovery is a call: a warrant whose act was lost is written again by asking +// warranted again, from the router's record. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"hand-act", "hand-acts", "warranted"} + +// Tool names. +const ( + HandActTool = Seat + ".hand-act" + HandActsTool = Seat + ".hand-acts" + WarrantedTool = Seat + ".warranted" +) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +// Tools are what the runtime serves. +func Tools(l *Log) []stdio.Tool { + return []stdio.Tool{ + {Name: HandActTool, + Description: "Record one act done by hand (novox/hq ADR 0565 §10): what was done, why, its cause in a word or a " + + "condition's kind (the word a second act for the same reason uses, which is how a repair done twice is " + + "found) and the condition it addresses. Who did it is the caller the runtime proves. verb is the act's " + + "verb as the log names it: a repair when not given, `hand-act drill` for a drill, or a recorder's own " + + "(`conditions silence`, `config set`). Said as act-recorded.", + Input: obj(map[string]any{"what": str("what was done"), "why": str("why, in a sentence"), + "cause": str("the cause, in a word or a condition's kind"), "condition": str("the condition's key, if any"), + "verb": str("the act's verb in the log; a repair when not given")}, "what", "why"), + Run: func(a map[string]any) (any, error) { + return l.HandAct(stdio.CallerOf(a), arg(a, "verb"), arg(a, "what"), arg(a, "why"), arg(a, "cause"), arg(a, "condition")) + }}, + {Name: HandActsTool, + Description: "What was done by hand lately, over both logs (this seat's and the controller's until its lease " + + "ends), newest first: each act with who, why, its cause and which log holds it, and the causes repaired by " + + "hand twice or more within a fortnight (a healer is wanted). Narrowed by cause and who; warrants=true lists " + + "every warrant recorded instead.", + Input: obj(map[string]any{"days": map[string]any{"type": "number", "description": "how many days back (default 14)"}, + "cause": str("only this cause"), "who": str("only acts whose who contains this"), + "warrants": str("\"true\": every warrant recorded, `.` with its act")}), + Run: func(a map[string]any) (any, error) { + if strings.EqualFold(arg(a, "warrants"), "true") || a["warrants"] == true { + return l.Warrants() + } + return l.HandActs(days(a), arg(a, "cause"), arg(a, "who")), nil + }}, + {Name: WarrantedTool, + Description: "Record an asker's act on the operator's warrant (ADR 0274, ADR 0259): read from the router's own " + + "record of that ask — who chose, through which channel, with which proofs and which answer — never from the " + + "caller, once per ask; refused for an ask still open, ended without a choice, or another asker's.", + Input: obj(map[string]any{"asker": str("the module that asked"), "ask": str("its ask's id")}, "asker", "ask"), + Run: func(a map[string]any) (any, error) { + return l.Warranted(stdio.CallerOf(a), arg(a, "asker"), arg(a, "ask")) + }}, + } +} + +// Warrants is every warrant recorded, by key. +func (l *Log) Warrants() (any, error) { + keys, err := l.Store.Keys(WarrantsState) + if err != nil { + return nil, fmt.Errorf("the warrants cannot be read: %w", err) + } + sort.Strings(keys) + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := l.Store.Get(WarrantsState, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{"warrants": rows, "count": len(rows)}, nil +} + +func arg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func days(a map[string]any) int { + switch v := a["days"].(type) { + case float64: + if v > 0 { + return int(v) + } + case string: + var n int + if _, err := fmt.Sscan(v, &n); err == nil && n > 0 { + return n + } + } + return 14 +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go new file mode 100644 index 00000000..64ec46d2 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -0,0 +1,405 @@ +//go:build integration + +package main + +// mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered +// from a real nats-server's key-value buckets, mesh/log.* from the log's real stream, mesh/publish onto its subjects). A recorder's hand act is kept in the +// log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info": + p.log(m) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +// log answers the runtime's log verbs from the log's real stream, LOG__ on mesh.log.... +func (p *player) log(m wire) { + var a struct { + Log string `json:"log"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + From uint64 `json:"from"` + } + _ = json.Unmarshal(m.Params, &a) + stream := "LOG_" + Module + "_" + a.Log + subject := "mesh.log." + Module + "." + a.Log + s, err := p.js.Stream(p.ctx, stream) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entry := func(r *jetstream.RawStreamMsg) map[string]any { + return map[string]any{"sequence": r.Sequence, "key": strings.TrimPrefix(r.Subject, subject+"."), + "value": json.RawMessage(r.Data), "at": r.Time} + } + switch m.Method { + case "mesh/log.append": + ack, err := p.js.Publish(p.ctx, subject+"."+a.Key, a.Value) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"sequence": ack.Sequence}, nil) + case "mesh/log.last": + r, err := s.GetLastMsgForSubject(p.ctx, subject+"."+a.Key) + if errors.Is(err, jetstream.ErrMsgNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, entry(r), nil) + case "mesh/log.read": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entries := []map[string]any{} + for seq := max(a.From, 1); seq <= info.State.LastSeq; seq++ { + if r, err := s.GetMsg(p.ctx, seq); err == nil { + entries = append(entries, entry(r)) + } + } + p.answer(m.ID, map[string]any{"entries": entries, "next": 0, "last": info.State.LastSeq}, nil) + case "mesh/log.info": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"entries": info.State.Msgs, "bytes": info.State.Bytes, + "max_bytes": info.Config.MaxBytes, "first": info.State.FirstSeq, "last": info.State.LastSeq}, nil) + } +} + +func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1953/L5-C2: an act recorded through the seat is kept on the bus and heard as act-recorded. +func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-hand-acts does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + // The log as the mesh makes it (ADR 0297): a stream kept by limits, refusing when full, never deleting. + logStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_" + Module + "_" + ActsLog, + Subjects: []string{"mesh.log." + Module + "." + ActsLog + ".>"}, MaxBytes: 64 << 20, Discard: jetstream.DiscardNew, + DenyDelete: true, DenyPurge: true, AllowDirect: true}) + if err != nil { + t.Fatal(err) + } + recorded, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + RecordedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(HandActTool, "ace.mesh-config", map[string]any{"what": "set dns on g14", "why": "it was wrong", + "cause": "config", "by": "someone else"}) + if err != nil { + t.Fatal(err) + } + var r Recorded + if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" { + t.Fatalf("hand-act answered %s", raw) + } + e, err := logStream.GetLastMsgForSubject(ctx, "mesh.log."+Module+"."+ActsLog+"."+r.Act.ID) + if err != nil || !strings.Contains(string(e.Data), `"by":"ace.mesh-config"`) { + t.Fatalf("the act is not kept in the log on the bus: %v", err) + } + msg, err := recorded.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) { + t.Fatalf("act-recorded was not said: %v", err) + } + // A warrant the router recorded, asked twice: one act. + asks, _ := js.KeyValue(ctx, "messenger_asks") + if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code", + "outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram", + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil { + t.Fatal(err) + } + for _, caller := range []string{"ace.claude-code", "g14.claude-code"} { + if _, err := p.call(WarrantedTool, caller, map[string]any{"asker": "claude-code", "ask": "a1"}); err != nil { + t.Fatal(err) + } + } + if info, err := logStream.Info(ctx); err != nil || info.State.Msgs != 2 { + t.Errorf("the log keeps %v, want the act and one warrant (%v)", info, err) + } + listed, err := p.call(HandActsTool, "person.jochen", map[string]any{}) + if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) { + t.Errorf("hand-acts answered %s, %v", listed, err) + } +} diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod new file mode 100644 index 00000000..df0875c3 --- /dev/null +++ b/modules/mesh-hand-acts/go.mod @@ -0,0 +1,22 @@ +module mesh-hand-acts + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum new file mode 100644 index 00000000..a0208db4 --- /dev/null +++ b/modules/mesh-hand-acts/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json new file mode 100644 index 00000000..96aff8b1 --- /dev/null +++ b/modules/mesh-hand-acts/module.json @@ -0,0 +1,65 @@ +{ + "module": "mesh-hand-acts", + "version": "1", + "runs-as": "mesh-hand-acts", + "claims": [ + { + "name": "mesh-hand-acts", + "scope": "mesh", + "serves": [ + "hand-act", + "hand-acts", + "warranted" + ] + } + ], + "reads": [ + "mesh-controller.hand-acts", + "messenger.asks" + ], + "logs": [ + { + "name": "acts", + "max-age-days": 400 + } + ], + "state": [ + "warrants" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-hand-acts", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-hand-acts", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-hand-acts" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-hand-acts", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-hand-acts", + "binary": "mesh-hand-acts", + "loads": [ + "mesh-hand-acts" + ] + } + ] + } +} diff --git a/modules/mesh-healers/.gitignore b/modules/mesh-healers/.gitignore new file mode 100644 index 00000000..0bafa572 --- /dev/null +++ b/modules/mesh-healers/.gitignore @@ -0,0 +1,2 @@ +/mesh-healers +/cmd/mesh-healers/mesh-healers diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md new file mode 100644 index 00000000..d00cdb1b --- /dev/null +++ b/modules/mesh-healers/README.md @@ -0,0 +1,46 @@ +# mesh-healers + +Holds the seat `mesh-healers` (novox/hq ADR 0564 §7, §8): the mesh's healers, moved out of the controller. Go, one +binary (`cmd/mesh-healers`), served by a runtime of its own on its own bus account (`runs-as`). + +## What it does + +It hears the condition keeper's `condition-raised` and `condition-changed` (seat `mesh-conditions`) on a durable +consumer. A condition whose kind a healer answers is acted on within the healer's budget, its settle and the mesh-wide +brake (twelve acts an hour). A repair is a call of the seat or tool that owns the act: H3 asks the grants part's +`grants_assert` for the module whose consumer is gone. H4 (the controller's own consumer) and H5 (the provider's own +repair) are registered and run elsewhere. + +Each act is begun by a create of its key in `attempts` before it is made and finished after, so a holder that dies +mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's +`tried`, resolver and escalation from `attempts`; a healer writes no condition. + +**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would +take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. + +Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared +`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits +are keys of `wakes` whose time to live is the wait: `settle.` ends with a settle and judges the condition +again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears +each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while +no instance watched is judged at the start, once the lease is first read. + +S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a +fortnight; a drill and a person's decision by design never count. + +## What it keeps + +- `mesh-healers_attempts`: every act and escalation, each key kept a week. +- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease, + each key kept a week. +- `mesh-healers_findings`: the brake, its own blindness, and S15. +- `mesh-healers_wakes`: the waits, each key living as long as it waits. + +## Its verbs and tools + +- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why; + `recount=true` counts the hand-act log again. +- `mesh-healers.heal`: one act now on an open condition, by key and why. +- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the + controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill. +- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go new file mode 100644 index 00000000..425eba47 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -0,0 +1,663 @@ +package main + +// The healers (novox/hq ADR 0564 §7, §8; ADR 0231): a known failure heals itself, under a brake, and every repair is +// said. The registry is compiled in; mesh-healers hears `condition-raised` and `condition-changed` of the seat +// mesh-conditions on a durable consumer, instead of looking every half minute. A repair is a call of the seat or tool +// that owns the act (H3: the grants part's grants_assert). Each act is begun by a create of its key in the bucket +// `attempts` before it is made and finished after, so a holder that dies mid-act has spent it and a second instance +// cannot begin the same act; budgets, settles and the mesh-wide brake count from there. Success is never the +// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`, +// the resolver and the escalation into the condition from `attempts`. +// +// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is +// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches +// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once +// the lease is handed away nothing is written there again. + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-healers" + Seat = "mesh-healers" +) + +// The states it keeps and reads, and the event it says. +const ( + AttemptsState = health.AttemptsState + WouldState = health.WouldState + FindingsState = health.FindingsState + LeaseRead = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey + ConditionsRead = "mesh-conditions.conditions" + HandActsRead = "mesh-controller.hand-acts" + ActedEvent = health.EventHealerActed + // ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays). + ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent + // RecordedEvent is the hand-act log's seat event, each act recorded there (ADR 0565 §11). + RecordedEvent = handacts.Seat + "." + handacts.EventRecorded + // The events it hears: the keeper's, on the seat mesh-conditions. + RaisedEvent = "mesh-conditions.condition-raised" + ChangedEvent = "mesh-conditions.condition-changed" + // AssertTool is the grants part's tool that asserts a module's buckets, logs and streams again. + AssertTool = "mesh-grants.grants_assert" +) + +// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour. +const ( + BrakeLimit = 12 + BrakeWindow = time.Hour +) + +// The outcomes an attempt has. +const ( + OutcomeActing = health.OutcomeActing + OutcomeActed = health.OutcomeActed + OutcomeFailed = health.OutcomeFailed + OutcomeEscalated = health.OutcomeEscalated +) + +// Store is the bus's buckets as this module reaches them, through its runtime. CreateTTL writes a key only when it has +// none, and says ErrExists otherwise; it and PutTTL give the key its own time to live, at whose end the bus removes it +// and a watch of the bucket hears it expire (a bucket declared key-ttl). +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + CreateTTL(state, key string, value any, ttl time.Duration) error + PutTTL(state, key string, value any, ttl time.Duration) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// KeptFor is how long an attempt and a would-act are kept: each key a week from its last write (ADR 0564 Limits). +const KeptFor = 7 * 24 * time.Hour + +// WakesState holds the healers' waits as keys whose time to live is the wait: `settle.` ends with a +// settle, `fortnight` when S15's oldest counted act leaves the fortnight. Its expiry, heard on the bucket's watch, is +// the wake: no timer runs in the process, and a restart loses nothing a key on the bus holds. +const WakesState = "wakes" + +// The wakes' keys. +const ( + settlePrefix = "settle." + fortnightKey = "fortnight" +) + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside. +type Condition struct { + Key string `json:"key"` + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity string `json:"severity"` + Resolver string `json:"resolver"` + Tried []json.RawMessage `json:"tried,omitempty"` +} + +// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until +// observation clears it. +func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 } + +// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key +// `..` is the healers' own), an act written to `would` while the lease names a +// controller (one key per healer and condition, `.`, its newest), and the body of healer-acted, +// epoch left out (ADR 0564 Limits 10). +type ( + Attempt = health.Attempt + Would = health.Would + Acted = health.HealerActed +) + +// Healer is one row of the registry. +type Healer struct { + ID string + Kinds []string + Condition string + Repair string + Then string + From string + Budget int + Window time.Duration + Settle time.Duration + // ActsIn is where the repair runs: here, or a module that repairs its own (H5), or the controller (H4). + ActsIn string + // applies says whether this condition is one it may act on, and what its budget counts against; why when not. + applies func(c Condition) (budget string, ok bool, why string) + // repair is the act: what it did, what came of it, or why it could not. + repair func(h *Healers, c Condition) (act, said string, err error) +} + +// actsHere marks a healer whose repair mesh-healers makes. +const actsHere = "mesh-healers" + +// Registry is to-be 45 §7's table, compiled in: every healer the mesh has, the ones mesh-healers runs with their act. +func Registry() []Healer { + return []Healer{ + {ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"}, + Condition: "a durable consumer of a module the bus does not hold (D6), or a seat's holder that does not answer (D3)", + Repair: "assert the module's buckets, logs and streams again through the grants part (grants_assert)", + Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)", + Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsHere, + applies: appliesToAModulesObject, repair: assertAgain}, + {ID: "H4", Kinds: []string{"consumer-behind"}, + Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable", + Repair: "re-make the consumer to deliver from now; the one marked resettable is the controller's own", + Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)", + Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, + ActsIn: "the controller, until its lease: the one consumer marked resettable is its own (ADR 0564 §7)"}, + {ID: "H5", Kinds: []string{"provider-failing"}, + Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing)", + Repair: "the provider repairs it itself through the server's own bootstrap command (ADR 0224 §5)", + Then: "announced failing by the provider, braked from ten minutes doubling to six hours", + From: "the identity provider's admin reset through its bootstrap command (179)", + Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute, + ActsIn: "the provider module (keycloak), ADR 0224 §5"}, + } +} + +// Healers is mesh-healers: the registry, what its acts reach, and what it holds. +type Healers struct { + Store Store + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) + Registry []Healer + + mu sync.Mutex + // lease is the controller's lease as last read: acting is on only while it is handed away. + acting bool + leaseWhy string + // declined is why each condition was last passed over, said once. + declined map[string]string + // acts are the hand-act log as watched, for S15. + acts map[string]HandAct +} + +// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564 +// §8, Limits 8, as ADR 0561 §3's reader reads it). +func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) { + on, why := buscompose.LeaseHandedAway(raw, found, readErr) + h.mu.Lock() + flipped := on != h.acting || h.leaseWhy == "" + h.acting, h.leaseWhy = on, leaseWords(on, why) + h.mu.Unlock() + if flipped { + h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy) + } +} + +// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting. +func leaseWords(on bool, why string) string { + if on { + return "the controller's lease is handed away: the healers act" + } + if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") { + return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would" + } + return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would" +} + +// Acting says whether the healers act, and why. +func (h *Healers) Acting() (bool, string) { + h.mu.Lock() + defer h.mu.Unlock() + if h.leaseWhy == "" { + return false, "the controller's lease has not been read yet: every act is written as would" + } + return h.acting, h.leaseWhy +} + +// healerFor is the healer that acts here on a kind. +func (h *Healers) healerFor(kind string) (Healer, bool) { + for _, r := range h.Registry { + if r.repair != nil && slices.Contains(r.Kinds, kind) { + return r, true + } + } + return Healer{}, false +} + +// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over. +func (h *Healers) Heard(body []byte) error { + var c Condition + if err := json.Unmarshal(body, &c); err != nil { + h.Logf("[%s] a condition event is not readable, and is passed over: %v", Module, err) + return nil + } + _, err := h.Judge(c, "") + return err +} + +// Judge is one condition judged by the healer registered for its kind. by names a person's `heal`, empty for an +// event. It answers what was done, in words. +func (h *Healers) Judge(c Condition, by string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + row, ok := h.healerFor(c.Kind) + if !ok { + return "no healer here answers the kind " + c.Kind, nil + } + if c.Escalated() { + return "it is handed to the operator already: no healer touches it until observation clears it", nil + } + budget, applies, why := row.applies(c) + if !applies { + h.decline(c.Key, row.ID, why) + return fmt.Sprintf("healer %s leaves it alone: %s", row.ID, why), nil + } + delete(h.declined, c.Key) + now := h.Now() + if !h.acting { + w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()} + if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil { + return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err) + } + return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil + } + all, err := h.attempts() + if err != nil { + h.blind(err) + return "", fmt.Errorf("the healers cannot read what they did, so they count no budget and act on nothing: %w", err) + } + h.see("blind", nil) + acts := actsWithin(all, now.Add(-BrakeWindow)) + if braked, said := h.brakeHolds(acts, now); braked { + h.see("brake", []Observation{brakeFinding(acts, said)}) + return "every healer has stopped: " + said, nil + } + h.see("brake", nil) + spent := spentOn(all, row, budget, now) + if n := len(spent); n > 0 && now.Sub(spent[n-1].Begun) < row.Settle { + h.wake(budget, spent[n-1].Begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s waits: its last act on it is still the observation's to judge until %s", row.ID, + spent[n-1].Begun.Add(row.Settle).UTC().Format(time.RFC3339)), nil + } + if len(spent) >= row.Budget { + return h.escalate(row, c, budget, spent) + } + if len(acts) >= BrakeLimit { + return "the brake holds from the next act on", nil + } + return h.act(row, c, budget, len(spent), by) +} + +func (h *Healers) whyNot() string { + if h.leaseWhy == "" { + return "the controller's lease has not been read yet" + } + return h.leaseWhy +} + +// act is one healer's act: begun by a create in attempts, made, finished, and said. +func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) { + begun := h.Now().UTC() + key := attemptKey(row.ID, c.Key, begun) + if by == "" { + by = "healer " + row.ID + } + window := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: row.Repair, Outcome: OutcomeActing, + Begun: begun} + if by != "healer "+row.ID { + a.Said = "asked by " + by + } + if err := h.Store.CreateTTL(AttemptsState, key, a, KeptFor); err != nil { + if errors.Is(err, ErrExists) { + return "another instance began this act first", nil + } + return "", fmt.Errorf("healer %s did not act on %s: its attempt could not be begun: %w", row.ID, c.Key, err) + } + act, said, err := row.repair(h, c) + a.Outcome = OutcomeActed + if err != nil { + a.Outcome, said = OutcomeFailed, err.Error() + } + if act != "" { + a.Act = act + } + finished := h.Now().UTC() + if a.Said != "" { + said = a.Said + "; " + said + } + a.Said, a.Finished = said+" ("+window+")", &finished + if err := h.Store.PutTTL(AttemptsState, key, a, KeptFor); err != nil { + h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window}) + h.wake(budget, begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s %s %s: %s — %s (%s)", row.ID, a.Outcome, c.Key, a.Act, said, window), nil +} + +// escalate is a spent budget: the condition is the operator's now. Written once as an attempt the keeper turns into +// the condition's resolver and urgency (ADR 0564 §7); an escalated condition is passed over until it clears. +func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attempt) (string, error) { + var tried []string + for _, s := range spent { + tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.Begun.UTC().Format("15:04 MST"), firstLine(s.Said))) + } + said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window, + c.Key, strings.Join(tried, "; ")) + now := h.Now().UTC() + window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator", + Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now} + if err := h.Store.CreateTTL(AttemptsState, attemptKey(row.ID, c.Key, now), a, KeptFor); err != nil && !errors.Is(err, ErrExists) { + return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated, + Said: said, Budget: window}) + return fmt.Sprintf("healer %s handed %s to the operator: %s", row.ID, c.Key, said), nil +} + +// tell says a heal as healer-acted. Not said is said in the log: the attempt holds it. +func (h *Healers) tell(e Acted) { + e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer + e.Show = "mesh-conditions.conditions key=" + e.Condition + if h.Publish == nil { + return + } + if err := h.Publish(ActedSubject, e, e.Healer+"."+e.Condition+"."+e.Outcome+"."+strconv.FormatInt(e.At.UnixNano(), 10)); err != nil { + h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err) + } +} + +// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has +// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still +// running keeps its key on the bus and needs nothing. +func (h *Healers) Rearm() { + all, err := h.attempts() + if err != nil { + h.Logf("[%s] what the healers did cannot be read, so no ended settle is judged: %v", Module, err) + return + } + now := h.Now() + newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first + for _, a := range all { + if a.Outcome != OutcomeEscalated { + newest[a.Healer+" "+a.BudgetKey] = a + } + } + var due []string + for _, a := range newest { + for _, r := range h.Registry { + if r.ID == a.Healer && r.repair != nil && !now.Before(a.Begun.Add(r.Settle)) && now.Sub(a.Begun) <= r.Window { + due = append(due, a.Condition) + } + } + } + sort.Strings(due) + for _, key := range due { + h.again(key) + } +} + +// wake writes the wait for a settle's end as a key whose time to live is the wait; its expiry judges the condition +// again (WakeHeard). Written again with the same end, it is the same wait. +func (h *Healers) wake(_ string, at time.Time, key string) { + ttl := at.Sub(h.Now()).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, settlePrefix+key, map[string]any{"condition": key, "at": at.UTC()}, ttl); err != nil { + h.Logf("[%s] the settle of %s is not written as a wait, so its end is judged at the next event: %v", Module, key, err) + } +} + +// WakeHeard is a change of the wakes bucket: an expiry is a wait ended — a settle's, whose condition is judged again, +// or S15's fortnight, counted again. +func (h *Healers) WakeHeard(key, op string) { + if op != OpExpire { + return + } + switch { + case key == fortnightKey: + h.Wanted() + case strings.HasPrefix(key, settlePrefix): + h.again(strings.TrimPrefix(key, settlePrefix)) + } +} + +// OpExpire is the watch's word for a key whose own time to live ended. +const OpExpire = stdio.OpExpire + +// again judges a condition again from the keeper's bucket, if it is still open. +func (h *Healers) again(key string) { + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil || !found { + return + } + if _, err := h.Judge(conditionOf(raw), ""); err != nil { + h.Logf("[%s] %s judged again at its settle's end: %v", Module, key, err) + } +} + +func conditionOf(raw []byte) Condition { + var c Condition + _ = json.Unmarshal(raw, &c) + return c +} + +// decline remembers why a healer passed a condition over, said once. +func (h *Healers) decline(key, healer, why string) { + if h.declined == nil { + h.declined = map[string]string{} + } + if h.declined[key] != why { + h.Logf("[%s] healer %s leaves %s alone: %s", Module, healer, key, why) + } + h.declined[key] = why +} + +func attemptKey(healer, condition string, begun time.Time) string { + return healer + "." + condition + "." + strconv.FormatInt(begun.UnixNano(), 10) +} + +// attempts are every attempt kept, oldest first. +func (h *Healers) attempts() ([]Attempt, error) { + keys, err := h.Store.Keys(AttemptsState) + if err != nil { + return nil, err + } + var out []Attempt + for _, k := range keys { + raw, found, err := h.Store.Get(AttemptsState, k) + if err != nil { + return nil, err + } + if !found { + continue + } + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the attempt %s is not readable: %w", k, err) + } + out = append(out, a) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Begun.Before(out[j].Begun) }) + return out, nil +} + +// actsWithin are the attempts since a moment that were acts: an escalation is a saying, and the brake does not count it. +func actsWithin(all []Attempt, since time.Time) []Attempt { + var out []Attempt + for _, a := range all { + if !a.Begun.Before(since) && a.Outcome != OutcomeEscalated { + out = append(out, a) + } + } + return out +} + +// spentOn is one healer's acts against one budget key within its window, oldest first. +func spentOn(all []Attempt, row Healer, budget string, now time.Time) []Attempt { + var out []Attempt + for _, a := range actsWithin(all, now.Add(-row.Window)) { + if a.Healer == row.ID && a.BudgetKey == budget { + out = append(out, a) + } + } + return out +} + +// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake already written +// and an act within the hour still — it lets go an hour after the last act, not the first. +func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) { + said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), BrakeWindow, BrakeLimit) + if len(acts) >= BrakeLimit { + return true, said + } + _, held, _ := h.Store.Get(FindingsState, brakeKey) + if held && len(acts) > 0 { + last := acts[len(acts)-1].Begun + return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said, + last.Add(BrakeWindow).UTC().Format("15:04 MST")) + } + return false, said +} + +var brakeKey = health.Key(ScopeMesh, "healers", "braked") + +// brakeFinding is the brake, as the urgent finding that says it. +func brakeFinding(acts []Attempt, said string) Observation { + counts := map[string]int{} + for _, a := range acts { + counts[a.Healer+" on "+a.Condition]++ + } + var most []string + for what, n := range counts { + most = append(most, fmt.Sprintf("%s ×%d", what, n)) + } + sort.Strings(most) + return Observation{Kind: "healers-braked", Token: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"}, + Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " + + "healer looping, not repairing — " + said, + Said: strings.Join(most, ", "), + Headline: "All automatic repairs stopped", + Needs: "read the details and decide whether the repairs may run again.", + Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " + + "like a loop, so all of them stopped.", + Resolved: "Automatic repairs run again"} +} + +// blind writes the healers' own blindness: they cannot read what they did. +func (h *Healers) blind(err error) { + h.see("blind", []Observation{{Kind: "probe-failed", Token: "failed", Severity: Warning, + Subject: Subject{Scope: ScopeProbe, ID: "healers"}, + Summary: "the healers cannot read what they did, so they count no budget and act on nothing", + Said: firstLine(err.Error()), Headline: "A check of the mesh could not run", + Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.", + Resolved: "The automatic repairs read what they did again"}}) +} + +// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it +// no longer sees (ADR 0564 §2). +func (h *Healers) see(source string, found []Observation) { + now := h.Now() + want := map[string]health.Finding{} + for _, o := range found { + o.Source = source + want[o.Key()] = o.Finding("", now) + } + keys, err := h.Store.Keys(FindingsState) + if err != nil { + h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err) + return + } + for _, k := range keys { + raw, found, err := h.Store.Get(FindingsState, k) + if err != nil || !found { + continue + } + was, err := health.DecodeFinding(raw) + if err != nil || was.Source != source { + continue + } + if f, still := want[k]; still { + if says(was) == says(f) { + delete(want, k) + } + continue + } + if err := h.Store.Delete(FindingsState, k); err != nil { + h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err) + } + } + for k, f := range want { + raw, err := health.EncodeFinding(f) + if err == nil { + err = h.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + h.Logf("[%s] the finding %s is not written: %v", Module, k, err) + } + } +} + +// --- H3 -------------------------------------------------------------------------------------------------------- + +// moduleOf is the module whose objects a condition names: a consumer of a module's own (`.`). The +// controller's own consumers are not a module's: the controller asserts them itself until its lease. +func moduleOf(c Condition) (string, string) { + stream, consumer, ok := strings.Cut(c.Subject.ID, ".") + if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" { + return "", "it names no consumer of a module whose objects the grants part asserts" + } + if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") { + return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease" + } + return consumer, "" +} + +// appliesToAModulesObject is H3's: a module's consumer the bus does not hold, its budget per condition. +func appliesToAModulesObject(c Condition) (string, bool, string) { + if _, why := moduleOf(c); why != "" { + return "", false, why + } + return c.Key, true, "" +} + +// assertAgain is H3: the grants part asserts the module's buckets, logs and streams again. +func assertAgain(h *Healers, c Condition) (string, string, error) { + module, why := moduleOf(c) + if module == "" { + return "", "", errors.New(why) + } + raw, err := h.Ask(AssertTool, map[string]any{"module": module}) + if err != nil { + return "", "", fmt.Errorf("the grants part did not assert %s: %w", module, err) + } + var a struct { + Says string `json:"says"` + } + _ = json.Unmarshal(raw, &a) + return "asserted " + module + "'s buckets, logs and streams again through the grants part", + orSaid(a.Says, "the grants part answered") + "; the probe that raised it says on its next run whether it is there", nil +} + +func orSaid(s, otherwise string) string { + if strings.TrimSpace(s) == "" { + return otherwise + } + return s +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go new file mode 100644 index 00000000..9b6f9dce --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -0,0 +1,496 @@ +package main + +import ( + "encoding/json" + "errors" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" +) + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + keysErr map[string]error + ttl map[string]time.Duration +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +// CreateTTL and PutTTL keep the key's time to live beside it, for a test to read. +func (s *memStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, true); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) PutTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, false); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) setTTL(state, key string, ttl time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ttl == nil { + s.ttl = map[string]time.Duration{} + } + s.ttl[state+"/"+key] = ttl +} + +// ttlOf is a key's time to live as last written; zero for none. +func (s *memStore) ttlOf(state, key string) time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return s.ttl[state+"/"+key] +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) keys(state string) []string { + k, _ := s.Keys(state) + return k +} + +func (s *memStore) attempt(t *testing.T, key string) Attempt { + t.Helper() + raw, _, _ := s.Get(AttemptsState, key) + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatalf("%s: %s", key, raw) + } + return a +} + +// world is mesh-healers over memory, with its clock, its asks, what it said and the wakes it set. +type world struct { + h *Healers + store *memStore + now time.Time + asked []string + said []Acted +} + +func newWorld() *world { + w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)} + w.fresh() + return w +} + +// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten. +func (w *world) fresh() { + w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, + Now: func() time.Time { return w.now }, + Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + w.asked = append(w.asked, key+" "+string(raw)) + return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil + }, + Publish: func(subject string, body any, _ string) error { + if a, ok := body.(Acted); ok && subject == "mesh.seat.mesh-healers.event.healer-acted" { + w.said = append(w.said, a) + } + return nil + }, + } +} + +func (w *world) handAway() { + w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) +} + +// lost is a module's consumer the bus does not hold, as D6 raises it. +func lost(module string) Condition { + return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self", + Subject: Subject{Scope: "bus", ID: "EVENTS." + module}} +} + +func body(c Condition) []byte { + raw, _ := json.Marshal(map[string]any{"key": c.Key, "kind": c.Kind, "subject": c.Subject, "severity": c.Severity, + "resolver": c.Resolver, "tried": c.Tried, "event": "condition-raised", "show": "mesh-conditions.conditions key=" + c.Key}) + return raw +} + +// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no +// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8). +func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) { + cases := []struct { + name string + lease func(h *Healers) + acts bool + }{ + {"never read", func(*Healers) {}, false}, + {"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false}, + {"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false}, + {"a controller holds it", func(h *Healers) { + h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil) + }, false}, + {"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true}, + } + for _, tc := range cases { + w := newWorld() + tc.lease(w.h) + if err := w.h.Heard(body(lost("records"))); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState) + if tc.acts { + if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) || + len(w.said) != 1 || w.said[0].Outcome != OutcomeActed { + t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said) + continue + } + a := w.store.attempt(t, attempts[0]) + if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" { + t.Errorf("%s: the attempt is %+v", tc.name, a) + } + continue + } + if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) { + t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said) + } + if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked { + t.Errorf("%s: a would-act moved the brake", tc.name) + } + } +} + +// An act waits out its settle, then its spent budget hands the condition to the operator once; an escalated condition +// is passed over; a second instance cannot begin an act already begun; the brake stops every healer past twelve acts +// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7). +func TestBudgetSettleEscalationAndBrake(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + settle := settlePrefix + c.Key + steps := []struct { + name string + after time.Duration + cond Condition + attempts int + outcome string // the newest said outcome, empty for none new + // how is what happens: an event, a restart over the same buckets, or the settle key's expiry heard. + how string + // wait is the settle key's time to live after the step, zero for no check. + wait time.Duration + }{ + {"acts: its settle is a key that expires", 0, c, 1, OutcomeActed, "event", 6*time.Minute + time.Second}, + {"within its settle: waits for the same end", 2 * time.Minute, c, 1, "", "event", 4*time.Minute + time.Second}, + {"a restart within its settle: the key on the bus still waits", time.Minute, c, 1, "", "restart", 4*time.Minute + time.Second}, + {"its key expires: judged again, budget spent, escalates", 4 * time.Minute, c, 2, OutcomeEscalated, "expire", 0}, + {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", "event", 0}, + } + for _, s := range steps { + w.now = w.now.Add(s.after) + before := len(w.said) + switch s.how { + case "restart": + w.fresh() + w.handAway() + w.h.Rearm() + case "expire": + w.h.WakeHeard(settle, OpExpire) + default: + if _, err := w.h.Judge(s.cond, ""); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + } + if n := len(w.store.keys(AttemptsState)); n != s.attempts { + t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts) + } + if s.wait != 0 && w.store.ttlOf(WakesState, settle) != s.wait { + t.Errorf("%s: the settle waits %s, want %s", s.name, w.store.ttlOf(WakesState, settle), s.wait) + } + switch { + case s.outcome == "" && len(w.said) != before: + t.Errorf("%s: said %+v", s.name, w.said[before:]) + case s.outcome != "" && (len(w.said) != before+1 || w.said[before].Outcome != s.outcome): + t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) + } + } + // Each attempt is kept a week from its last write. + for _, k := range w.store.keys(AttemptsState) { + if ttl := w.store.ttlOf(AttemptsState, k); ttl != KeptFor { + t.Errorf("the attempt %s is kept %s, want a week", k, ttl) + } + } + // A settle whose key expired while no instance watched is judged at the start. + r := newWorld() + r.handAway() + _ = r.store.Put(ConditionsRead, c.Key, c) + if _, err := r.h.Judge(c, ""); err != nil { + t.Fatal(err) + } + r.now = r.now.Add(7 * time.Minute) + r.fresh() + r.handAway() + r.h.Rearm() + if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated { + t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said) + } + // A second instance cannot begin the act an attempt's key already holds. + x := newWorld() + x.handAway() + _ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor) + if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 { + t.Errorf("a second instance acted: %q, asked %v", said, x.asked) + } + // The brake: twelve acts in the hour stop every healer, said as an urgent finding. + b := newWorld() + b.handAway() + for i := 0; i < BrakeLimit; i++ { + b.now = b.now.Add(time.Minute) + _ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x", + Outcome: OutcomeActed, Begun: b.now}, KeptFor) + } + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 { + t.Errorf("past the brake: %q, asked %v", said, b.asked) + } + raw, braked, _ := b.store.Get(FindingsState, brakeKey) + f, _ := health.DecodeFinding(raw) + if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module { + t.Errorf("the brake is written as %s", raw) + } + b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") { + t.Errorf("the brake let go before an hour after the last act: %q", said) + } + b.now = b.now.Add(6 * time.Minute) + if _, err := b.h.Judge(lost("mail"), ""); err != nil || len(b.asked) != 1 { + t.Errorf("an hour after the last act the healers do not act again: %v, asked %v", err, b.asked) + } + if _, braked, _ := b.store.Get(FindingsState, brakeKey); braked { + t.Error("the brake's finding outlived the brake") + } + // Blind to what they did, they act on nothing and say so. + z := newWorld() + z.handAway() + z.store.keysErr[AttemptsState] = errors.New("the bus is away") + if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 { + t.Errorf("blind, the healers acted: %v", err) + } + if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) { + t.Errorf("blind, the findings are %v", keys) + } +} + +// H3 answers a module's consumer the bus does not hold, never the controller's own or a condition naming no consumer; +// the registry holds every healer the mesh has, H3 run here, H4 and H5 run elsewhere (ADR 0231 §2, ADR 0564 §7). +func TestTheRegistryAndWhatH3Answers(t *testing.T) { + cases := []struct { + c Condition + holds bool + }{ + {lost("records"), true}, + {lost("controller"), false}, + {lost("controller-merges"), false}, + {Condition{Key: "seat.x.silent", Kind: "holder-silent", Subject: Subject{Scope: "seat", ID: "issue-tracker"}}, false}, + {Condition{Key: "bus.EVENTS.records.behind", Kind: "consumer-behind", Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}, false}, + } + for _, tc := range cases { + w := newWorld() + w.handAway() + said, _ := w.h.Judge(tc.c, "") + if acted := len(w.asked) == 1; acted != tc.holds { + t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said) + } + } + runs := map[string]bool{} + for _, r := range Registry() { + if len(r.Kinds) == 0 || r.Budget < 1 || r.Window <= 0 || r.Settle <= 0 || r.Repair == "" || r.ActsIn == "" { + t.Errorf("%s is not a whole row: %+v", r.ID, r) + } + runs[r.ID] = r.repair != nil + } + if !runs["H3"] || runs["H4"] || runs["H5"] || len(runs) != 3 { + t.Errorf("the healers run here are %v, want H3 alone of H3, H4, H5", runs) + } +} + +// S15: a cause repaired by hand twice within a fortnight is a finding; a decision by design never counts; it clears +// when fewer than two remain, and the healers wake when the oldest leaves the fortnight (ADR 0564 Limits 2). +func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { + w := newWorld() + act := func(key, verb, cause string, ago time.Duration) { + raw, _ := json.Marshal(HandAct{ID: key, At: w.now.Add(-ago), By: "person.jochen", Verb: verb, Why: "x", Cause: cause}) + w.h.HandActHeard(key, "put", raw) + } + act("a1", "broker consumer-reset", "consumer-behind", 10*24*time.Hour) + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Fatalf("one act is a finding: %v", keys) + } + // The second of the cause is the seat's own act, heard as act-recorded: one in each log counts two (ADR 0565 §11). + seatAct, _ := json.Marshal(HandAct{ID: "a2", At: w.now.Add(-24 * time.Hour), By: "person.jochen", + Verb: "broker consumer-reset", Why: "x", Cause: "consumer-behind"}) + w.h.ActRecorded(seatAct) + // A drill in each log counts none. + act("a3", "hand-act drill", "drill", time.Hour) + drill, _ := json.Marshal(HandAct{ID: "a4", At: w.now.Add(-2 * time.Hour), Verb: "hand-act drill", Why: "x", Cause: "drill"}) + w.h.ActRecorded(drill) + act("a5", "conditions silence", "operator-answer", time.Hour) + act("a6", "conditions silence", "operator-answer", time.Hour) + act("a7", "hand-act record", "relogin", 20*24*time.Hour) + act("a8", "hand-act record", "relogin", time.Hour) + want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"} + if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) { + t.Fatalf("findings %v, want %v", keys, want) + } + raw, _, _ := w.store.Get(FindingsState, want[0]) + if !strings.Contains(string(raw), "healer H4 answers this cause") { + t.Errorf("the finding does not say its healer is not enough: %s", raw) + } + if ttl := w.store.ttlOf(WakesState, fortnightKey); ttl != 4*24*time.Hour+time.Second { + t.Errorf("the fortnight waits %s, want the oldest act's leaving it in 4 days", ttl) + } + w.now = w.now.Add(4*24*time.Hour + time.Second) + w.h.Wanted() + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Errorf("past the fortnight the finding stays: %v", keys) + } + w.h.HandActHeard("a5", "delete", nil) + if len(w.h.acts) != 7 { + t.Errorf("a deleted entry is kept: %d", len(w.h.acts)) + } +} + +// The verbs: heal acts on an open condition by key and says why it was asked; drill waits for the log's home; the reads +// answer every bucket. +func TestTheVerbsAndReads(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil { + t.Error("heal took no why") + } + if _, err := w.h.Heal("bus.none", "x", "person.jochen"); err == nil { + t.Error("heal acted on no open condition") + } + if _, err := w.h.Heal(c.Key, "it is gone again", "person.jochen"); err != nil { + t.Fatal(err) + } + keys := w.store.keys(AttemptsState) + if len(keys) != 1 || !strings.Contains(w.store.attempt(t, keys[0]).Said, "asked by person.jochen (it is gone again)") { + t.Errorf("heal's attempt is %v", keys) + } + answer, err := w.h.Answer(7, true) + if err != nil { + t.Fatal(err) + } + a := answer.(map[string]any) + if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 { + t.Errorf("healers answered %v", a) + } + // drill records into mesh-hand-acts as a drill; while that seat has no holder, into the controller's drill until + // the lease is handed away (ADR 0565 §11). + drills := []struct { + name string + seatGone bool + leaseAway bool + args map[string]any + log, err string + }{ + {"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""}, + {"no holder: the controller's", true, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogController, ""}, + {"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "handed its lease away"}, + {"says nothing", false, true, map[string]any{}, "", "says what"}, + } + for _, d := range drills { + x := newWorld() + if d.leaseAway { + x.handAway() + } else { + x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil) + } + var asked []string + x.h.Ask = func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if d.seatGone && strings.HasPrefix(key, "seat:mesh-hand-acts.") { + return nil, errors.New("nats: no responders available for request") + } + return json.RawMessage(`{}`), nil + } + var got any + var err error + for _, tool := range Tools(x.h) { + if tool.Name == DrillTool { + got, err = tool.Run(d.args) + } + } + switch { + case d.err != "": + if err == nil || !strings.Contains(err.Error(), d.err) { + t.Errorf("drill %s: %v", d.name, err) + } + case err != nil || got.(map[string]any)["log"] != d.log: + t.Errorf("drill %s: %v, %v", d.name, got, err) + case d.log == handacts.LogSeat && !strings.Contains(asked[0], `"verb":"hand-act drill"`): + t.Errorf("drill %s asked %v", d.name, asked) + } + } + if _, err := w.h.read(WouldState, ""); err != nil { + t.Error(err) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go new file mode 100644 index 00000000..6e48bfba --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -0,0 +1,118 @@ +// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its +// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun, +// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds +// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's +// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and watches the +// controller's hand-act log for S15. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) PutTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).PutTTL(key, value, ttl) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state, key string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch(key, handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + +func main() { + h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} + // The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended + // while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry. + var started sync.Once + go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { + h.LeaseHeard(c.Value, c.Op == "put", nil) + started.Do(func() { go h.Rearm() }) + return nil + }) + go watched(WakesState, "", func(c stdio.StateChange) error { + h.WakeHeard(c.Key, c.Op) + return nil + }) + go watched(HandActsRead, "", func(c stdio.StateChange) error { + h.HandActHeard(c.Key, c.Op, c.Value) + return nil + }) + // S15 over both logs: the seat's act-recorded on a durable consumer, beside the controller's bucket's watch. + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(RecordedEvent, func(e stdio.Envelope) error { + h.ActRecorded(e.Body) + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, RecordedEvent, err, wait) + } + }() + for _, event := range []string{RaisedEvent, ChangedEvent} { + go func(event string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(event, func(e stdio.Envelope) error { return h.Heard(e.Body) }) + if err == nil { + logf("[%s] hears %s", Module, event) + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, event, err, wait) + } + }(event) + } + if err := stdio.Serve("", Tools(h)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go new file mode 100644 index 00000000..af3745b9 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1, +// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's +// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each. +// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + for _, c := range []struct { + name string + got, want []string + }{ + {"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil}, + {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}}, + {"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-controller.drill"}}, + {"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}}, + } { + if !slices.Equal(c.got, c.want) { + t.Errorf("%s %v, want %v", c.name, c.got, c.want) + } + } + states := map[string]bool{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + KeyTTL bool `json:"key-ttl"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.KeyTTL + } + // Each key of attempts and would lives a week from its last write (KeptFor, would has no other deleter), and each + // wait in wakes as long as it waits: their buckets take a key's own time to live. + if !states[AttemptsState] || !states[WouldState] || !states[WakesState] || states[FindingsState] { + t.Errorf("the states take a key's own time to live as %v, want attempts, would and wakes alone", states) + } + readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool, + WakesState: WakesTool, "a settle's end": HealTool, + "an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool} + tools := map[string]bool{} + for _, tool := range Tools(&Healers{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + if name, _, _ := strings.Cut(tool, " "); !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/observation.go b/modules/mesh-healers/cmd/mesh-healers/observation.go new file mode 100644 index 00000000..853550b5 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/observation.go @@ -0,0 +1,77 @@ +package main + +// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their +// blindness and S15, each written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the +// controller's key for the same condition. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is what saw it: the brake, the healers' blindness, or S15. + Source string +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} diff --git a/modules/mesh-healers/cmd/mesh-healers/reads-outside b/modules/mesh-healers/cmd/mesh-healers/reads-outside new file mode 100644 index 00000000..cdc54f27 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-healers/module.json diff --git a/modules/mesh-healers/cmd/mesh-healers/tools.go b/modules/mesh-healers/cmd/mesh-healers/tools.go new file mode 100644 index 00000000..d3452b95 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/tools.go @@ -0,0 +1,230 @@ +package main + +// The seat's verbs and the module's own reads (novox/hq ADR 0564 §7, §12): `healers` is the registry, what the healers +// did and would do, the brake and whether they act; `heal` takes one act now on a condition, within budget and brake; +// `drill` waits for the hand-act log's home; and a read of each bucket it keeps, so a recovery is a call. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"healers", "heal", "drill"} + +// Tool names. +const ( + HealersTool = Seat + ".healers" + HealTool = Seat + ".heal" + DrillTool = Seat + ".drill" + FindingsTool = "healers_findings" + WouldTool = "healers_would" + WakesTool = "healers_wakes" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(h *Healers) []stdio.Tool { + return []stdio.Tool{ + {Name: HealersTool, + Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " + + "days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " + + "would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " + + "again for S15 (a cause repaired by hand twice in a fortnight).", + Input: obj(map[string]any{ + "days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"}, + "recount": map[string]any{"type": "boolean", "description": "count the hand-act log again now"}}), + Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }}, + {Name: HealTool, + Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " + + "event would have it — or, while the controller holds its lease, the act it would take written as would. " + + "Says what was done.", + Input: obj(map[string]any{ + "key": map[string]any{"type": "string", "description": "the condition's key"}, + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}), + Run: func(a map[string]any) (any, error) { + return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a)) + }}, + {Name: DrillTool, + Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with what it " + + "tests, in the hand-act log of the seat mesh-hand-acts (the controller's drill while that seat has no " + + "holder and the controller holds its lease). A drill is never a repair: S15 never counts it.", + Input: obj(map[string]any{"what": map[string]any{"type": "string", "description": "what was done on purpose"}, + "why": map[string]any{"type": "string", "description": "what the drill tests"}, + "condition": map[string]any{"type": "string", "description": "the condition it is meant to raise, if any"}}), + Run: func(a map[string]any) (any, error) { + return h.Drill(strArg(a, "what"), strArg(a, "why"), strArg(a, "condition")) + }}, + {Name: FindingsTool, + Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " + + "each cause repaired by hand twice in a fortnight (S15); or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }}, + {Name: WakesTool, + Description: "What the healers wait for (their bucket wakes): each settle still running (`settle.`) " + + "and S15's fortnight end (`fortnight`), each a key whose time to live is the wait and whose expiry wakes them; " + + "or one whole by key. heal acts on a condition now, and healers with recount=true counts S15 now.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }}, + {Name: WouldTool, + Description: "What the healers would have done while the controller holds its lease (their bucket would, each " + + "key kept a week): one per healer and condition, its newest; or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }}, + } +} + +// Drill records a drill (ADR 0565 §11, which ends ADR 0564 Limits 1's refusal): through the seat mesh-hand-acts as +// the act `hand-act drill`, and while that seat has no holder that answers and the controller still holds its lease, +// through the controller's own `drill`, which records it as a drill too. It answers which log took it. +func (h *Healers) Drill(what, why, condition string) (any, error) { + if what == "" || why == "" { + return nil, errors.New("a drill says what was done on purpose and what it tests: nothing was recorded") + } + if h.Ask == nil { + return nil, errors.New("the hand-act log cannot be reached from here: nothing was recorded") + } + body := map[string]any{"what": what, "why": why, "verb": handacts.VerbDrill} + controller := map[string]any{"what": what, "why": why} + if condition != "" { + body["condition"], controller["condition"] = condition, condition + } + answer, err := h.Ask("seat:"+handacts.Seat+".hand-act", body) + if err == nil { + return map[string]any{"log": handacts.LogSeat, "answer": answer}, nil + } + if !handacts.NoResponder(err) { + return nil, fmt.Errorf("%s.hand-act refused the drill: %w", handacts.Seat, err) + } + if acting, _ := h.Acting(); acting { + return nil, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing was "+ + "recorded (%w)", handacts.Seat, err) + } + answer, cerr := h.Ask("seat:mesh-controller.drill", controller) + if cerr != nil { + return nil, fmt.Errorf("%s has no holder that answers (%v), and the controller's drill refused too: %w", handacts.Seat, + err, cerr) + } + return map[string]any{"log": handacts.LogController, "answer": answer}, nil +} + +// Heal is the verb heal: one condition judged now by its healer. +func (h *Healers) Heal(key, why, caller string) (any, error) { + if key == "" || why == "" { + return nil, errors.New("heal names the condition's key and says why") + } + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil { + return nil, fmt.Errorf("the open conditions cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("no condition is open under %s", key) + } + said, err := h.Judge(conditionOf(raw), caller+" ("+why+")") + if err != nil { + return nil, err + } + return map[string]any{"said": said}, nil +} + +// Answer is the verb healers. +func (h *Healers) Answer(days int, recount bool) (any, error) { + if recount { + h.Wanted() + } + now := h.Now() + all, err := h.attempts() + if err != nil { + return nil, fmt.Errorf("what the healers did cannot be read: %w", err) + } + var recent []Attempt + for _, a := range all { + if now.Sub(a.Begun) <= time.Duration(days)*24*time.Hour { + recent = append(recent, a) + } + } + sort.SliceStable(recent, func(i, j int) bool { return recent[i].Begun.After(recent[j].Begun) }) + braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now) + var rows []map[string]any + for _, r := range h.Registry { + rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair, + "budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle), + "then": r.Then, "acts-in": r.ActsIn, "from": r.From}) + } + acting, why := h.Acting() + would, err := h.read(WouldState, "") + if err != nil { + would = map[string]any{"unread": err.Error()} + } + h.mu.Lock() + declined := map[string]string{} + for k, v := range h.declined { + declined[k] = v + } + h.mu.Unlock() + if recent == nil { + recent = []Attempt{} + } + return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would, + "brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()}, + "declined": declined, + "note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil +} + +// read is every key of a bucket with its value, or one whole. +func (h *Healers) read(state, key string) (any, error) { + if key != "" { + raw, found, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("nothing is kept under %s in %s", key, state) + } + return json.RawMessage(raw), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := h.Store.Get(state, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{state: rows, "count": len(rows)}, nil +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go new file mode 100644 index 00000000..065f2c41 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -0,0 +1,166 @@ +package main + +// S15: a cause repaired by hand twice within a fortnight is a healer wanted (ADR 0231 §27, §28; ADR 0564 Limits 2). +// The hand-act log is the controller's bucket mesh-controller_hand-acts, a key-value bucket with no events: mesh-healers +// watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the +// finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design +// (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the +// fortnight: the key `fortnight` of the healers' wakes expires when the oldest counted act leaves it. From the log's home (issue 1953) they +// watch that instead. + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// Within is how far back a repeated cause counts. +const Within = 14 * 24 * time.Hour + +// HandAct is what S15 reads of one entry of the controller's hand-act log (mesh-controller internal/link HandAct). +type HandAct struct { + ID string `json:"id"` + At time.Time `json:"at"` + By string `json:"by"` + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` + Why string `json:"why"` + Cause string `json:"cause"` + Kind string `json:"kind,omitempty"` +} + +// decided are the verbs whose entries are a person's decision by design, with the causes they hold for (every cause +// when none): the controller's handActVerbs, read as S15 reads them. A verb not listed counts as a repair. +var decided = map[string][]string{ + "warrant": nil, + "seat": {"seat-passed"}, + "conditions silence": {"operator-answer"}, + "hand-act record": {"drill"}, + "secret issued": nil, + "hand-act drill": nil, + "retire approve": nil, + "retire reject": nil, + "cleanup delete": nil, + "dead-letters deliver": nil, + "dead-letters drop": nil, + "collect": nil, + "mirrors": nil, + "move-placement": nil, + "move-config": nil, + "bus upgrade": nil, + "upgrade release-backlog": nil, + "secret accept": nil, + "secret rotate": {"leaked-in-logs"}, + "rotate": {"leaked-in-logs"}, +} + +// personsDecision is whether an act is a person's decision by design, by the verb that recorded it. +func personsDecision(a HandAct) bool { + causes, ok := decided[a.Verb] + if !ok { + return false + } + return len(causes) == 0 || slices.Contains(causes, a.Cause) +} + +// ActRecorded is one act the seat mesh-hand-acts recorded, heard as act-recorded: counted beside the controller's +// log's, under a key of its own log, so an act is counted once whichever log holds it (ADR 0565 §11). +func (h *Healers) ActRecorded(body []byte) { + var a HandAct + if json.Unmarshal(body, &a) != nil || a.ID == "" { + return + } + h.HandActHeard(handacts.LogSeat+":"+a.ID, "put", body) +} + +// HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again. +func (h *Healers) HandActHeard(key, op string, value []byte) { + h.mu.Lock() + if h.acts == nil { + h.acts = map[string]HandAct{} + } + if op != "put" { + delete(h.acts, key) + } else { + var a HandAct + if json.Unmarshal(value, &a) == nil { + h.acts[key] = a + } + } + h.mu.Unlock() + h.Wanted() +} + +// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, +// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. +func (h *Healers) Wanted() []Observation { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + byCause := map[string][]HandAct{} + for _, a := range h.acts { + if personsDecision(a) || now.Sub(a.At) > Within { + continue + } + byCause[a.Cause] = append(byCause[a.Cause], a) + } + var causes []string + for c, acts := range byCause { + if len(acts) >= 2 { + causes = append(causes, c) + } + } + sort.Strings(causes) + var found []Observation + var next time.Time + for _, cause := range causes { + acts := byCause[cause] + sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) + var lines []string + for _, a := range acts { + lines = append(lines, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"), + strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why)) + } + newest := acts[len(acts)-1] + wanted := "a healer is wanted for it" + for _, r := range h.Registry { + if slices.Contains(r.Kinds, cause) { + wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its budget "+ + "is not enough", r.ID) + } + } + words := strings.NewReplacer(".", " ", "_", " ").Replace(cause) + found = append(found, Observation{Kind: "healer-wanted", Token: "healer-wanted", Severity: Warning, + Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause}, + Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts), + int(Within.Hours()/24), newest.By, wanted), + Said: strings.Join(lines, "; "), + Headline: capital(words) + " keeps being fixed by hand", + Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+ + "for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)), + Resolved: "Resolved: no more hand repairs of " + words}) + if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) { + next = leaves + } + } + h.see("S15", found) + if !next.IsZero() { + ttl := next.Sub(now).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil { + h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err) + } + } + return found +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go new file mode 100644 index 00000000..db056254 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -0,0 +1,423 @@ +//go:build integration + +package main + +// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's +// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An +// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the +// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the +// hour the brake stops every healer, said as an urgent finding on the bus. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + asked []string +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +// deliver hands the bundle one event as the runtime does from its consumer. +func (p *player) deliver(key string, body []byte) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("e-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{ + "key": key, "node": "ace", "body": json.RawMessage(body)}}}) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &ask) + p.mu.Lock() + p.asked = append(p.asked, ask.Key+" "+string(ask.Body)) + p.mu.Unlock() + if ask.Key != AssertTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + p.answer(m.ID, map[string]any{"says": "1 the bus lacks: created at this ask"}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +func raised(key, module string) []byte { + raw, _ := json.Marshal(map[string]any{"key": key, "kind": "consumer-lost", "severity": "warning", "resolver": "self", + "subject": map[string]any{"scope": "bus", "id": "EVENTS." + module}, "event": "condition-raised", + "show": health.ShowKeeper(key)}) + return raw +} + +// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes. +func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-healers does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState, + "mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + bucket := func(name string) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + t.Fatal(err) + } + return kv + } + keys := func(name string) []string { + k, _ := bucket(name).Keys(ctx) + return k + } + lease := bucket("mesh-controller_lease") + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil { + t.Fatal(err) + } + acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + healers := func() map[string]any { + raw, err := p.call(HealersTool, map[string]any{}) + if err != nil { + return nil + } + var a map[string]any + _ = json.Unmarshal(raw, &a) + return a + } + eventually(t, "the lease read as a controller's", func() bool { + a := healers() + return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller") + }) + + // The controller holds the lease: the induced failure is written as would, never acted on. + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 }) + if k := keys(Module + "_" + AttemptsState); len(k) != 0 { + t.Fatalf("acted under the controller's lease: %v", k) + } + // The write that hands the lease away: the same failure is acted on. + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "the healers acting", func() bool { return healers()["acting"] == true }) + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the act begun and finished on the bus", func() bool { + k := keys(Module + "_" + AttemptsState) + if len(k) != 1 { + return false + } + e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0]) + return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`) + }) + msg, err := acted.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) { + t.Fatalf("healer-acted was not said: %v", err) + } + p.mu.Lock() + asked := strings.Join(p.asked, "\n") + p.mu.Unlock() + if !strings.Contains(asked, AssertTool+` {"module":"records"}`) { + t.Errorf("the grants part was asked %s", asked) + } + // Eleven more acts within the hour: the twelfth was the limit, and the next condition meets the brake. + attempts := bucket(Module + "_" + AttemptsState) + for i := 0; i < BrakeLimit-1; i++ { + begun := time.Now().Add(-time.Duration(i+1) * time.Minute).UTC() + raw, _ := json.Marshal(Attempt{Healer: "H3", Condition: fmt.Sprintf("bus.EVENTS.m%d.missing", i), BudgetKey: "x", + Outcome: OutcomeActed, Begun: begun}) + if _, err := attempts.Put(ctx, attemptKey("H3", fmt.Sprintf("bus.EVENTS.m%d.missing", i), begun), raw); err != nil { + t.Fatal(err) + } + } + p.deliver(RaisedEvent, raised("bus.EVENTS.mail.missing", "mail")) + eventually(t, "the brake written as a finding on the bus", func() bool { + e, err := bucket(Module+"_"+FindingsState).Get(ctx, brakeKey) + if err != nil { + return false + } + f, err := health.DecodeFinding(e.Value()) + return err == nil && f.Severity == health.Urgent && f.Kind == "healers-braked" + }) + if n := len(keys(Module + "_" + AttemptsState)); n != BrakeLimit { + t.Errorf("past the brake the attempts are %d, want %d", n, BrakeLimit) + } +} diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod new file mode 100644 index 00000000..6351b3a4 --- /dev/null +++ b/modules/mesh-healers/go.mod @@ -0,0 +1,22 @@ +module mesh-healers + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum new file mode 100644 index 00000000..a0208db4 --- /dev/null +++ b/modules/mesh-healers/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json new file mode 100644 index 00000000..634dd388 --- /dev/null +++ b/modules/mesh-healers/module.json @@ -0,0 +1,82 @@ +{ + "module": "mesh-healers", + "version": "1", + "runs-as": "mesh-healers", + "claims": [ + { + "name": "mesh-healers", + "scope": "mesh", + "serves": [ + "healers", + "heal", + "drill" + ] + } + ], + "consumes": [ + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed", + "mesh-hand-acts.act-recorded" + ], + "invokes": [ + "mesh-grants.grants_assert", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-controller.drill" + ], + "reads": [ + "mesh-controller.lease", + "mesh-conditions.conditions", + "mesh-controller.hand-acts" + ], + "state": [ + { + "name": "attempts", + "key-ttl": true + }, + { + "name": "would", + "key-ttl": true + }, + "findings", + { + "name": "wakes", + "key-ttl": true + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-healers", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-healers", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-healers" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-healers", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-healers", + "binary": "mesh-healers", + "loads": [ + "mesh-healers" + ] + } + ] + } +} diff --git a/modules/mesh-provisioner/.gitignore b/modules/mesh-provisioner/.gitignore new file mode 100644 index 00000000..b51e2c5c --- /dev/null +++ b/modules/mesh-provisioner/.gitignore @@ -0,0 +1,2 @@ +/mesh-provisioner +/cmd/mesh-provisioner/mesh-provisioner diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/busread.go b/modules/mesh-provisioner/cmd/mesh-provisioner/busread.go new file mode 100644 index 00000000..5de3c58c --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/busread.go @@ -0,0 +1,204 @@ +package main + +// The bus as the holder composes from it (frombus.Reader), read through this module's runtime with its own `reads`: +// placement, each node's record in its own have key, the indexer's modules and manifests, the builder's builds and +// mesh-config's resolved values. A plan names no value, so no own secret and no credential is read (compose.Planning). + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// busReader reads the bus for frombus through a Store. +type busReader struct { + s Store + fetch func(ctx context.Context, url string) ([]byte, error) +} + +func (r busReader) each(read string, visit func(key string, value []byte)) error { + keys, err := r.s.Keys(read) + if err != nil { + return fmt.Errorf("%s: %w", read, err) + } + for _, k := range keys { + e, err := r.s.Get(read, k) + if err != nil { + return fmt.Errorf("%s %s: %w", read, k, err) + } + if e != nil { + visit(k, e.Value) + } + } + return nil +} + +func (r busReader) Placements(context.Context) (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + err := r.each(PlacementRead, func(k string, v []byte) { + node, err := delivery.ParsePlacementKey(k) + if err != nil { + return + } + if p, err := delivery.DecodePlacement(v); err == nil && p.Node == node { + out[node] = p + } + }) + return out, err +} + +func (r busReader) Records(context.Context) (map[string]delivery.NodeRecord, error) { + out := map[string]delivery.NodeRecord{} + err := r.each(HaveRead, func(k string, v []byte) { + node, _, isNode, err := delivery.ParseHaveKey(k) + if err != nil || !isNode { + return + } + if h, err := delivery.DecodeHaveNode(v); err == nil && h.Record != nil { + out[node] = *h.Record + } + }) + return out, err +} + +func (r busReader) Modules(context.Context) (map[string]delivery.Module, error) { + out := map[string]delivery.Module{} + err := r.each(ModulesRead, func(k string, v []byte) { + module, err := delivery.ParseModulesKey(k) + if err != nil { + return + } + if m, err := delivery.DecodeModule(v); err == nil { + out[module] = m + } + }) + return out, err +} + +func (r busReader) Manifest(_ context.Context, digest string) ([]byte, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return nil, err + } + e, err := r.s.Get(ManifestsRead, key) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("the indexer holds no manifest %s", digest) + } + m, err := delivery.DecodeManifest(e.Value) + if err != nil { + return nil, err + } + return []byte(m.JSON), nil +} + +func (r busReader) Values(_ context.Context, node, module string) (map[string]any, error) { + var out map[string]any + prefix := node + "." + module + "." + err := r.each(ResolvedRead, func(k string, v []byte) { + name, ok := strings.CutPrefix(k, prefix) + if !ok || strings.Contains(name, ".") { + return + } + if value, laid := catalogue.LaidValue(v); laid { + if out == nil { + out = map[string]any{} + } + out[name] = value + } + }) + return out, err +} + +func (busReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return nil, nil } + +func (busReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + return nil, nil, nil +} + +func (r busReader) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return nil, err + } + e, err := r.s.Get(BuildsRead, key) + if err != nil || e == nil { + return nil, err + } + b, err := delivery.DecodeBuilderBuild(e.Value) + if err != nil { + return nil, err + } + return &b, nil +} + +// archiveWithin bounds one read of a templates archive. +const archiveWithin = 30 * time.Second + +func (r busReader) Archive(ctx context.Context, url, digest string) ([]byte, error) { + if r.fetch == nil { + return nil, fmt.Errorf("this holder reads no archive (%s)", url) + } + fetching, cancel := context.WithTimeout(ctx, archiveWithin) + defer cancel() + body, err := r.fetch(fetching, url) + if err != nil { + return nil, err + } + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got) + } + return body, nil +} + +func (r busReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) { + all, err := r.Values(ctx, node, module) + var out map[string]any + for _, k := range keys { + if v, ok := all[k]; ok { + if out == nil { + out = map[string]any{} + } + out[k] = v + } + } + return out, err +} + +// httpFetch reads an archive from the artifact store over HTTP. +func httpFetch(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s", url, resp.Status) + } + return io.ReadAll(io.LimitReader(resp.Body, 256<<20)) +} + +// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names. +func FromBus(s Store) Composer { + r := busReader{s: s, fetch: httpFetch} + return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go new file mode 100644 index 00000000..a97e47c7 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -0,0 +1,673 @@ +package main + +// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed +// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them +// in its states `plans` (`.`) and `provisions` (`..[.]`), which it alone +// writes. +// +// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the +// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants +// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a +// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5). +// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that +// consumer, `_bound` under the same key: present is provided (the vault's generation carried into the +// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by +// itself: a key of a provider that writes no outcome stays asked. +// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's +// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it; +// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The states it keeps, beside Registry, Needs, Applied and Collect. +const ( + ProvisionsState = provisions.ProvisionsState + PlansState = provisions.PlansState +) + +// The reads it composes from. +const ( + PlacementRead = delivery.PlacementRead + WantsRead = delivery.AssignerModule + ".wants" + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" + BuildsRead = delivery.BuilderModule + ".builds" + HaveRead = "node-engine.have" + ResolvedRead = "mesh-config.resolved" +) + +// Providers are the providing modules that keep an entry per consumer, `_bound`, read as `.bound`: the +// outcome of each provision they make. A provider not named here keeps none. +var Providers = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"} + +// boundRead is a provider's entries as this module reads them. +func boundRead(module string) string { return module + ".bound" } + +// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names +// (module → fingerprint), the newest build of every other placed module. +type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) + +// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's +// record of it. +type Asker interface { + Publish(subject string, body any, id string) error + Record(id string) ([]byte, bool, error) +} + +// Holder is the seat's holder. +type Holder struct { + Store Store + Compose Composer + Asks Asker + Now func() time.Time + Logf func(format string, args ...any) + + mu sync.Mutex +} + +// Composed is what one node's composition did. +type Composed struct { + Node string `json:"node"` + Plans []string `json:"plans-written,omitempty"` + Gone []string `json:"plans-removed,omitempty"` + Changed []string `json:"provisions-changed,omitempty"` + Asked []string `json:"retiring-asked,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +// ComposeNode composes one node: its plans and its provisions. +func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) { + h.mu.Lock() + defer h.mu.Unlock() + out := Composed{Node: node} + wanted, err := h.wantedOn(node) + if err != nil { + return out, err + } + atWants, err := h.Compose(ctx, node, wanted) + if err != nil { + return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) + } + newest, err := h.Compose(ctx, node, nil) + if err != nil { + return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err) + } + if err := h.writePlans(node, atWants, &out); err != nil { + out.Problems = append(out.Problems, err.Error()) + } + needed := map[string]provisions.Provision{} + for _, plans := range []map[string]catalogue.Plan{atWants, newest} { + for module, plan := range plans { + for _, p := range plan.Provisions { + pv := provisions.Provision{Provision: p.Provision, Local: p.Local, + Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: p.ProviderModule, Node: p.Provider}} + key, err := pv.Key() + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if _, seen := needed[key]; !seen { + needed[key] = pv + } + } + } + } + h.judgeNode(node, needed, &out) + return out, nil +} + +// wantedOn is the fingerprint each want on a node names, by module. +func (h *Holder) wantedOn(node string) (map[string]string, error) { + out := map[string]string{} + keys, err := h.Store.Keys(WantsRead) + if err != nil { + return nil, fmt.Errorf("the wants cannot be read: %w", err) + } + for _, k := range keys { + n, module, err := delivery.ParseWantKey(k) + if err != nil || n != node { + continue + } + e, err := h.Store.Get(WantsRead, k) + if err != nil || e == nil { + continue + } + if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" { + out[module] = w.FP + } + } + return out, nil +} + +// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. +func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { + var problems []error + for _, module := range sortedKeys(plans) { + key, err := provisions.PlanKey(node, module) + if err != nil { + problems = append(problems, err) + continue + } + raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]}) + if err != nil { + problems = append(problems, err) + continue + } + if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) { + continue + } + if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil { + problems = append(problems, fmt.Errorf("the plan %s: %w", key, err)) + continue + } + out.Plans = append(out.Plans, key) + } + keys, err := h.Store.Keys(PlansState) + if err != nil { + return errors.Join(append(problems, err)...) + } + for _, k := range keys { + n, module, ok := strings.Cut(k, ".") + if !ok || n != node { + continue + } + if _, still := plans[module]; still { + continue + } + if err := h.Store.Delete(PlansState, k); err != nil { + problems = append(problems, err) + continue + } + out.Gone = append(out.Gone, k) + } + return errors.Join(problems...) +} + +// judgeNode brings the node's provision keys to what is needed and what each provider said. +func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) { + held := h.held(node) + for _, key := range unionKeys(needed, held) { + want, isNeeded := needed[key] + have, isHeld := held[key] + var next provisions.Provision + switch { + case isNeeded: + next = want + if isHeld { + next.Reference = have.Reference + } + h.outcome(key, &next) + case have.State == provisions.Retiring: + next = have + if next.Ask == "" && next.Warrant == nil { + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if gone, why := h.retired(key, have); gone { + if err := h.Store.Delete(ProvisionsState, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Changed = append(out.Changed, key+" (retired: "+why+")") + } + continue + } + default: + next = have + next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+ + "any more", node, have.Provision, have.Provider.Module, have.Provider.Node) + next.Warrant = nil + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if isHeld && same(have, next) { + continue + } + next.At = h.Now().UTC() + raw, err := provisions.Encode(next) + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err)) + continue + } + out.Changed = append(out.Changed, key+" "+next.State) + } +} + +// held is the node's provision keys as kept. +func (h *Holder) held(node string) map[string]provisions.Provision { + out := map[string]provisions.Provision{} + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return out + } + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + if p, err := provisions.Decode(e.Value); err == nil { + out[k] = p + } + } + return out +} + +// same says two provisions say the same, their time aside. +func same(a, b provisions.Provision) bool { + a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0 + ra, _ := json.Marshal(a) + rb, _ := json.Marshal(b) + return bytes.Equal(ra, rb) +} + +// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what +// the holder needs and nothing more. +type entry struct { + // State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits. + State string `json:"state,omitempty"` + Why string `json:"why,omitempty"` + Refused string `json:"refused,omitempty"` + Waits string `json:"waits,omitempty"` + VaultName string `json:"vault-name,omitempty"` + Generation int `json:"generation,omitempty"` +} + +// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has +// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key +// asked, said so. +func (h *Holder) outcome(key string, p *provisions.Provision) { + p.Ask, p.Warrant = "", nil + if !keepsEntries(p.Provider.Module) { + h.haveOutcome(p) + return + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil { + p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "" + return + } + var en entry + if err := json.Unmarshal(e.Value, &en); err != nil { + p.State, p.Why = provisions.Asked, "the provider's entry is not readable" + return + } + switch { + case en.State == provisions.Failed: + p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed") + case en.State == provisions.Waits: + p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person") + case en.State == "" && en.Refused != "": + p.State, p.Why = provisions.Failed, en.Refused + case en.State == "" && en.Waits != "": + p.State, p.Why = provisions.Waits, en.Waits + case en.State != "" && en.State != provisions.Provided: + p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is" + return + default: + p.State, p.Why = provisions.Provided, "" + if en.VaultName != "" || en.Generation > 0 { + p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation} + } + } +} + +// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's +// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is +// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run. +func (h *Holder) haveOutcome(p *provisions.Provision) { + where := p.Provider.Module + " on " + p.Provider.Node + key := p.Provider.Node + "." + p.Provider.Module + e, err := h.Store.Get(HaveRead, key) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet" + return + } + have, err := delivery.DecodeHave(e.Value) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable" + return + } + wantFP := "" + if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil { + if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed { + wantFP = w.FP + } + } + switch { + case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed: + p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed") + case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP): + p.State, p.Why = provisions.Provided, "" + default: + p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+ + "health %s)", where, have.Phase, have.Health) + } +} + +func keepsEntries(module string) bool { + for _, m := range Providers { + if m == module { + return true + } + } + return false +} + +// retired says a retiring key the operator approved is retired: its provider's entry is gone. +func (h *Holder) retired(key string, p provisions.Provision) (bool, string) { + if p.Warrant == nil || p.Warrant.Option != OptionApprove { + return false, "" + } + if !keepsEntries(p.Provider.Module) { + return false, "" + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil || e != nil { + return false, "" + } + return true, "its provider removed its credential on the operator's warrant" +} + +// ---- the retiring ask ---------------------------------------------------------------------------------------------- + +// The options a retiring ask offers. +const ( + OptionApprove = "approve" + OptionDecline = "decline" +) + +// askLasts is how long a retiring ask stays open. +const askLasts = asks.ApproveLasts - 10*time.Minute + +// retiringAsk is the ask of one retiring provision, composed from the key alone. +func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask { + what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node, + p.Consumer.Module, p.Consumer.Node) + a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?", + Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+ + "is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node, + p.Provision, p.Provider.Module, what), + Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes", + About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"}, + {OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"}, + } { + binds, _ := asks.ActDigest(retireAct(key, id, o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +func retireAct(key, id, option string) asks.Act { + return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option} +} + +// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands. +func (h *Holder) ask(p *provisions.Provision, key string) error { + if p.Ask != "" { + return nil + } + if h.Asks == nil { + return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed") + } + now := h.Now().UTC() + id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix()) + a := retiringAsk(key, *p, now, id) + if err := a.Check(now); err != nil { + return err + } + if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil { + return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err) + } + p.Ask = id + return nil +} + +// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the +// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes. +func (h *Holder) Decide() []string { + h.mu.Lock() + defer h.mu.Unlock() + if h.Asks == nil { + return nil + } + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return []string{"the provisions cannot be read: " + err.Error()} + } + var said []string + for _, k := range keys { + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + p, err := provisions.Decode(e.Value) + if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil { + continue + } + raw, found, err := h.Asks.Record(p.Ask) + if err != nil || !found { + continue + } + var rec struct { + State string `json:"state"` + Ask *asks.Ask `json:"ask"` + Warrant *asks.Warrant `json:"warrant"` + } + if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != p.Ask { + continue + } + option := "" + if w.Outcome == asks.OutcomeChosen { + o, err := w.For(Module, *rec.Ask) + if err == nil { + err = o.Performs(retireAct(k, p.Ask, o.ID)) + } + if err != nil { + said = append(said, k+": the answer is refused: "+err.Error()) + continue + } + option = o.ID + } else { + // Ended with no choice: asked again at the next change. + p.Ask = "" + h.put(k, p) + said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes") + continue + } + by := "the operator" + if w.By != nil { + by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) + } + p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()} + h.put(k, p) + said = append(said, k+": "+option+" by "+by) + } + return said +} + +func (h *Holder) put(key string, p provisions.Provision) { + p.At = h.Now().UTC() + if raw, err := provisions.Encode(p); err == nil { + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + h.Logf("[%s] %s is not written: %v", Module, key, err) + } + } +} + +// ---- what a change touches -------------------------------------------------------------------------------------- + +// NodesOfPlacement is the node a placement key names. +func NodesOfPlacement(key string) []string { + if n, err := delivery.ParsePlacementKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfWant is the node a want key names. +func NodesOfWant(key string) []string { + if n, _, err := delivery.ParseWantKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change +// of the module's manifest touches. +func (h *Holder) NodesOfModule(module string) []string { + nodes := map[string]bool{} + if keys, err := h.Store.Keys(PlacementRead); err == nil { + for _, k := range keys { + n, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + e, err := h.Store.Get(PlacementRead, k) + if err != nil || e == nil { + continue + } + if p, err := delivery.DecodePlacement(e.Value); err == nil { + for _, m := range p.Modules { + if m.Module == module { + nodes[n] = true + } + } + } + } + } + if keys, err := h.Store.Keys(ProvisionsState); err == nil { + for _, k := range keys { + if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil { + if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module { + nodes[p.Consumer.Node] = true + } + } + } + } + return sortedKeys(nodes) +} + +// NodeOfEntry is the consumer's node a provider's entry key names. +func NodeOfEntry(key string) []string { + if n, _, _, _, err := provisions.ParseKey(key); err == nil { + return []string{n} + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A +// retiring key whose provider removed its entry on the operator's warrant goes. +func (h *Holder) Rejudge(key string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + e, err := h.Store.Get(ProvisionsState, key) + if err != nil || e == nil { + return "", err + } + have, err := provisions.Decode(e.Value) + if err != nil { + return "", err + } + if have.State == provisions.Retiring { + if gone, why := h.retired(key, have); gone { + return key + " retired: " + why, h.Store.Delete(ProvisionsState, key) + } + return "", nil + } + next := have + h.outcome(key, &next) + if same(have, next) { + return "", nil + } + h.put(key, next) + return key + " " + next.State, nil +} + +// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at +// the start. +func (h *Holder) RetireNeeds() (int, error) { + keys, err := h.Store.Keys(Needs) + if err != nil { + return 0, err + } + n := 0 + for _, k := range keys { + if err := h.Store.Delete(Needs, k); err != nil { + return n, err + } + n++ + } + return n, nil +} + +// orSaid is the first of the words that says something. +func orSaid(words ...string) string { + for _, w := range words { + if strings.TrimSpace(w) != "" { + return w + } + } + return "" +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go new file mode 100644 index 00000000..4c319ee1 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// router is the operator's channel as the holder asks on it. +type router struct { + asked map[string]asks.Ask + records map[string][]byte +} + +func (r *router) Publish(_ string, body any, id string) error { + r.asked[id] = body.(asks.Ask) + return nil +} + +func (r *router) Record(id string) ([]byte, bool, error) { + raw, ok := r.records[id] + return raw, ok, nil +} + +// answer is the operator's answer to an ask. +func (r *router) answer(t *testing.T, id, option string) { + t.Helper() + a := r.asked[id] + o, ok := a.Option(option) + if !ok { + t.Fatalf("the ask %s offers no %s", id, option) + } + w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, + Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram", + At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()} + r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w}) +} + +// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted +// builds and at the newest. +type mesh struct { + h *Holder + s *memStore + r *router + atWant map[string]map[string]catalogue.Plan + atNewest map[string]map[string]catalogue.Plan + composed []string +} + +func newMesh() *mesh { + m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}}, + atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}} + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + m.composed = append(m.composed, node) + if keptAt != nil { + return m.atWant[node], nil + } + return m.atNewest[node], nil + }} + return m +} + +// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set). +func planOf(provs ...catalogue.PlanProvision) catalogue.Plan { + return catalogue.Plan{Provisions: provs} +} + +var ( + pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"} + oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"} + split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"} +) + +func (m *mesh) state(t *testing.T, key string) provisions.Provision { + t.Helper() + e, _ := m.s.Get(ProvisionsState, key) + if e == nil { + return provisions.Provision{} + } + p, err := provisions.Decode(e.Value) + if err != nil { + t.Fatal(err) + } + return p +} + +func (m *mesh) compose(t *testing.T, node string) Composed { + t.Helper() + c, err := m.h.ComposeNode(context.Background(), node) + if err != nil { + t.Fatal(err) + } + return c +} + +// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry +// alone; the vault's generation reaches the reference; a +// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2). +func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { + m := newMesh() + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)} + m.atNewest["g14"] = m.atWant["g14"] + key := "g14.gitea.postgres-database" + steps := []struct { + name string + entry string // the provider's entry written before the step, "-" to delete + state string + why string + }{ + {"composed: asked", "", provisions.Asked, ""}, + {"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"}, + {"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"}, + {"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"}, + {"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""}, + } + for _, s := range steps { + if s.entry != "" { + m.s.write(t, "postgres.bound", key, s.entry) + } + m.compose(t, "g14") + p := m.state(t, key) + if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" { + t.Errorf("%s: %+v", s.name, p) + } + } + if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 { + t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference) + } + // A provider making no credential per consumer reports through its own have (clause 3): never provided by the + // holder alone. + split := "g14.gitea.split-dns" + for _, row := range []struct{ have, want, state, why string }{ + {"", "", provisions.Asked, "says nothing yet"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""}, + } { + if row.have != "" { + m.s.write(t, HaveRead, "g14.systemd-resolved", row.have) + } + if row.want != "" { + m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want) + } + m.compose(t, "g14") + if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) { + t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state) + } + } + if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { + t.Error("the plan is not written") + } + if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 { + t.Errorf("a quiet pass wrote %+v", c) + } + // An entry's change judges its key alone, no composition. + before := len(m.composed) + m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`) + if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before { + t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before) + } +} + +// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed +// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the +// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again; +// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3). +func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { + m := newMesh() + pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client" + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want + m.s.write(t, "postgres.bound", pg, `{}`) + steps := []struct { + name string + change func() + asked []string + states map[string]string + removed bool + }{ + {"a key composed ahead of its want is asked, not retiring", func() {}, nil, + map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false}, + {"one requirement removed and another added: only the removed retires", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + }, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false}, + {"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false}, + {"needed again before the warrant: back to its outcome", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)} + }, nil, map[string]string{pg: provisions.Provided}, false}, + {"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} }, + []string{pg}, map[string]string{pg: provisions.Retiring}, false}, + } + for _, s := range steps { + s.change() + c := m.compose(t, "g14") + if !slices.Equal(c.Asked, s.asked) { + t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked) + } + for k, st := range s.states { + if p := m.state(t, k); p.State != st { + t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why) + } + } + } + p := m.state(t, pg) + a := m.r.asked[p.Ask] + if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") || + !strings.Contains(a.Explanation, "data stay") { + t.Errorf("the ask does not say what goes from which provider on which node: %+v", a) + } + // Nothing before the warrant: the provider's entry stands and the key stays. + if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil { + t.Errorf("decided before an answer: %v", said) + } + m.r.answer(t, p.Ask, OptionApprove) + m.h.Decide() + if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") { + t.Fatalf("the warrant is not in the key: %+v", w) + } + if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring { + t.Errorf("retired while the provider's entry stands: %q", said) + } + _ = m.s.Delete("postgres.bound", pg) + if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" { + t.Errorf("not retired once the provider removed its entry: %q", said) + } + // A decline is kept, and nothing is asked again while nothing changes. + d := newMesh() + d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + d.compose(t, "g14") + d.atWant["g14"] = map[string]catalogue.Plan{} + d.compose(t, "g14") + d.r.answer(t, d.state(t, pg).Ask, OptionDecline) + d.h.Decide() + if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline { + t.Errorf("a decline was asked again: %+v", c) + } +} + +// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1). +func TestTheHolderHoldsNoTimer(t *testing.T) { + files, _ := filepath.Glob("*.go") + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" && + slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) { + t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name) + } + } + return true + }) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go new file mode 100644 index 00000000..30a16446 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -0,0 +1,104 @@ +package main + +// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and +// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The holder's tools. +const ( + ProvisionsTool = "provisioner_provisions" + PlansTool = "provisioner_plans" + ComposeTool = "provisioner_compose" + DecideTool = "provisioner_decide" +) + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +func holderTools(h *Holder) []stdio.Tool { + arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: ProvisionsTool, + Description: "Every provision the seat keeps (novox/hq ADR 0570): its key ..[.], " + + "provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " + + "state, with how many keys and bytes the state holds; or one key whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"), + "node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}}, + Run: func(a map[string]any) (any, error) { + return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state")) + }}, + {Name: PlansTool, + Description: "Every module's provisioning plan the seat composed, ., narrowed by node, with how many " + + "keys and bytes the state holds; or one plan whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, ."), + "node": str("a node")}}, + Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }}, + {Name: ComposeTool, + Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " + + "was written, removed, changed and asked.", + Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")}, + "required": []string{"node"}}, + Run: func(a map[string]any) (any, error) { + if arg(a, "node") == "" { + return nil, fmt.Errorf("compose names the node") + } + return h.ComposeNode(context.Background(), arg(a, "node")) + }}, + {Name: DecideTool, + Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " + + "key, as the router's word on an ask would.", + Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }}, + } +} + +// read is a state's keys narrowed, or one whole, with the state's fill. +func (h *Holder) read(state, key, node, want string) (any, error) { + if key != "" { + e, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("%s holds no %s", state, key) + } + return json.RawMessage(e.Value), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := []map[string]any{} + bytes := 0 + for _, k := range keys { + e, err := h.Store.Get(state, k) + if err != nil || e == nil { + continue + } + bytes += len(e.Value) + if node != "" && !strings.HasPrefix(k, node+".") { + continue + } + row := map[string]any{"key": k} + if state == ProvisionsState { + p, err := provisions.Decode(e.Value) + if err != nil { + row["unreadable"] = err.Error() + } else { + if want != "" && p.State != want { + continue + } + row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why + } + } + rows = append(rows, row) + } + return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 3955516d..583072b0 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -1,23 +1,28 @@ // mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR -// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR -// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its -// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its -// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this -// module says, it says on stderr. +// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259 +// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change +// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and +// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its +// start. stdout is the MCP channel; what this module says, it says on stderr. package main import ( + "context" "fmt" "os" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. const ( - Seat = "mesh-provisioner" - Verb = "provisioning" + Seat = "mesh-provisioner" + Module = "mesh-provisioner" + Verb = "provisioning" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state). func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the +// router's record of one read under that name. +type runtimeAsks struct{} + +func (runtimeAsks) Publish(subject string, body any, id string) error { + return stdio.SeatPublish(subject, body, id) +} + +func (runtimeAsks) Record(id string) ([]byte, bool, error) { + e, err := stdio.SeatRecord("", id) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake. +type work struct { + mu sync.Mutex + pending map[string]bool + wake chan struct{} +} + +func (w *work) add(nodes ...string) { + if len(nodes) == 0 { + return + } + w.mu.Lock() + for _, n := range nodes { + w.pending[n] = true + } + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *work) run(h *Holder) { + for range w.wake { + w.mu.Lock() + nodes := sortedKeys(w.pending) + w.pending = map[string]bool{} + w.mu.Unlock() + for _, n := range nodes { + c, err := h.ComposeNode(context.Background(), n) + switch { + case err != nil: + logf("[%s] %s is not composed: %v", Module, n, err) + case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0: + logf("[%s] composed %s: %+v", Module, n, c) + } + } + } +} + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch("", handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + func main() { - go publishing(runtimeStore{}) - if err := stdio.Serve("", Tools(runtimeStore{})); err != nil { + s := runtimeStore{} + h := &Holder{Store: s, Compose: FromBus(s), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} + w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)} + go w.run(h) + go starting(h) + go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil }) + go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil }) + go watched(ModulesRead, func(c stdio.StateChange) error { + if module, err := delivery.ParseModulesKey(c.Key); err == nil { + w.add(h.NodesOfModule(module)...) + } + return nil + }) + for _, provider := range Providers { + go watched(boundRead(provider), func(c stdio.StateChange) error { + if said, err := h.Rejudge(c.Key); err != nil { + logf("[%s] %s is not judged again: %v", Module, c.Key, err) + } else if said != "" { + logf("[%s] %s", Module, said) + } + return nil + }) + } + go watched(Registry, func(c stdio.StateChange) error { + if c.Op != "put" { + if _, _, err := Publish(s); err != nil { + logf("[%s] the registry is not written back: %v", Module, err) + } + } + return nil + }) + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error { + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) + } + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait) + } + }() + if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil { logf("%v", err) os.Exit(1) } } -// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or -// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first -// pass is tried again until it lands. -func publishing(s Store) { - for wait := 2 * time.Second; ; { - time.Sleep(wait) - written, deleted, err := Publish(s) - switch { - case err != nil: - logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait) - wait = min(wait*2, time.Minute) - continue - case len(written) > 0 || len(deleted) > 0: - logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted) +// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers, +// each tried again until it lands. +func starting(h *Holder) { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + written, deleted, err := Publish(h.Store) + if err == nil { + if len(written)+len(deleted) > 0 { + logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted) + } + break } - wait = time.Hour + logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait) + time.Sleep(wait) + } + if n, err := h.RetireNeeds(); err != nil { + logf("[%s] the retired needs are not all deleted: %v", Module, err) + } else if n > 0 { + logf("[%s] deleted %d key(s) of the retired needs", Module, n) + } + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index 4661b226..6d48aadb 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -106,9 +106,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) { for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect, "shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect, "contribution": "mesh-provisioner_" + Collect, - "dir": "mesh-controller_assignments", "access": "mesh-controller_assignments", - "port": "mesh-controller_assignments", "seat": "mesh-controller_assignments", - "machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} { + "dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans", + "port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans", + "machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} { i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind }) if TheRegistry[i].Bucket != bucket { t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket) @@ -288,6 +288,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { Serves []string `json:"serves"` } `json:"claims"` State []string `json:"state"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` OwnSecrets map[string]string `json:"own-secrets"` } if err := json.Unmarshal(raw, &m); err != nil { @@ -300,7 +302,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { !slices.Equal(m.Claims[0].Serves, []string{Verb}) { t.Errorf("the claim: %+v", m.Claims) } - if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) { + // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } + // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools + // its own and the composition it does on a change. + served := map[string]bool{} + for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) { + served[tool.Name] = true + } + for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb, + Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool, + "a composition on a change": ComposeTool, "a decision on an answer": DecideTool} { + if !served[tool] { + t.Errorf("%s has no tool: %s is not served", what, tool) + } + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which is not served", tool) + } + } + for _, r := range []string{PlacementRead, WantsRead, ModulesRead, ManifestsRead, BuildsRead, HaveRead, ResolvedRead} { + if !slices.Contains(m.Reads, r) { + t.Errorf("the manifest does not read %s, which the composition reads", r) + } + } + for _, p := range Providers { + if !slices.Contains(m.Reads, boundRead(p)) { + t.Errorf("the manifest does not read %s's entries", p) + } + } + if slices.Contains(m.Reads, "mesh-controller.assignments") { + t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)") + } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside new file mode 100644 index 00000000..203aee31 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-provisioner/module.json diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go index 7d7d6f3e..15f0bb3f 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go @@ -48,16 +48,15 @@ type Answer struct { Decision string `json:"decision"` } -// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's -// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which -// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in -// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration, -// which the node-engine keeps for root alone. +// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR +// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state +// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for +// root alone, nor the controller's assignments. func placement(kind, what string) Answer { - return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments", + return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState, Key: ".", - Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." + - "placement)", Decision: "novox/hq ADR 0349 clause 6"} + Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " + + "(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"} } // oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1). diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index 9f0b6d60..b380be43 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -2,4 +2,4 @@ module mesh-provisioner go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index bb6beaa1..d2e21659 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 650425b3..1aba09af 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -11,11 +11,36 @@ ] } ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-assigner.placement", + "mesh-delivery-assigner.wants", + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.manifests", + "mesh-delivery-builder.builds", + "node-engine.have", + "mesh-config.resolved", + "gitea.bound", + "keycloak.bound", + "postgres.bound", + "route-adapter.bound", + "mesh-vault.bound" + ], + "tools": [ + "provisioner_provisions", + "provisioner_plans", + "provisioner_compose", + "provisioner_decide" + ], "state": [ "registry", "needs", "applied", - "collect" + "collect", + "provisions", + "plans" ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/modules/mesh-self-check/.gitignore b/modules/mesh-self-check/.gitignore new file mode 100644 index 00000000..2c7bac8b --- /dev/null +++ b/modules/mesh-self-check/.gitignore @@ -0,0 +1,2 @@ +/mesh-self-check +/cmd/mesh-self-check/mesh-self-check diff --git a/modules/mesh-self-check/README.md b/modules/mesh-self-check/README.md new file mode 100644 index 00000000..9411474c --- /dev/null +++ b/modules/mesh-self-check/README.md @@ -0,0 +1,33 @@ +# mesh-self-check + +Holds the seat `mesh-self-check` (novox/hq ADR 0564 §6): the mesh's self-check, moved out of the controller. Go, one +binary (`cmd/mesh-self-check`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes its buckets. + +## What it does + +Every five minutes (the first run a minute after it starts) it runs its registry of probes, each within thirty +seconds. A probe is one live invariant of a design, with its assertion as the controller had it, reading its facts on +the bus (the bus module's tools for the bus's own state). A probe whose facts have no home on the bus yet answers +`deferred`, naming them: never a pass. A probe that cannot run is said in the verdict at once, and written as a finding +of its own (`probe..failed`) when the next run cannot run it either. + +What a probe finds is written as a finding, one key `..` per fault, only when what it says changes, and +deleted when a run no longer sees it. The condition keeper (`mesh-conditions`) raises and clears conditions from them. +A finding one look can be wrong about (a consumer behind after a burst) is written when two runs in a row see it. + +Every run ends with the event `checked`, the body of the controller's `doctor-heartbeat`, which the watchman (S10) and +mesh-watcher on another machine hear: a self-check that stops checking is said through a path that does not pass +through it. + +## What it keeps + +- `mesh-self-check_findings`: what it finds wrong now. +- `mesh-self-check_verdict`: the key `last`, the last run whole. +- `mesh-self-check_verdicts`: every run (each probe's findings at most twenty), kept a week by the bucket's age. + +## Its verbs and tools + +- `mesh-self-check.doctor`: the last verdict; `run=true` runs now; `probe=` one probe whole. +- `mesh-self-check.probes`: the registry, with what each deferred probe lacks. +- `self_check_findings`, `self_check_verdicts`: the reads of its buckets. diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go new file mode 100644 index 00000000..4225eead --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -0,0 +1,57 @@ +// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds +// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts` +// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is +// the MCP channel; what this module says, it says on stderr. +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// NewChecker is the self-check over the runtime. +func NewChecker() *Checker { + return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} +} + +func main() { + c := NewChecker() + go func() { + <-stdio.Served() + c.Keep(context.Background()) + }() + if err := stdio.Serve("", Tools(c)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go new file mode 100644 index 00000000..fb245a78 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves +// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts, +// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by +// itself a tool that starts it by hand (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + m := readManifest(t) + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + // checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 { + t.Errorf("emits %v", m.Emits) + } + if !slices.Equal(m.Invokes, []string{StreamsTool, ClosedTool, SignalsVerb}) { + t.Errorf("invokes %v, want what the probes ask", m.Invokes) + } + if len(m.Reads) != 0 { + t.Errorf("reads %v: the self-check reads its facts through the bus module's tools", m.Reads) + } + states := map[string]int{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + TTL int `json:"ttl-seconds"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.TTL + } + if states[VerdictsState] != 7*24*60*60 { + t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState]) + } + // ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool, + "the run": DoctorTool + " run=true"} + tools := map[string]bool{} + for _, tool := range Tools(&Checker{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + name, _, _ := cutSpace(tool) + if !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} + +func cutSpace(s string) (string, string, bool) { + for i, r := range s { + if r == ' ' { + return s[:i], s[i+1:], true + } + } + return s, "", false +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/observation.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go new file mode 100644 index 00000000..71209ce9 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/observation.go @@ -0,0 +1,142 @@ +package main + +// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an +// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong +// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key +// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is the probe that saw it, or the self-check itself for a probe that could not run. + Source string + // Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277). + Confirm bool +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} + +// worded gives a finding the operator's words registered for its kind, where it says none of its own. +func worded(f Observation) Observation { + w, ok := words[f.Kind] + if !ok { + return f + } + if f.Headline == "" { + f.Headline = w.Headline + } + if f.Explanation == "" { + f.Explanation = w.Explanation + } + if f.Resolved == "" { + f.Resolved = w.Resolved + } + if f.Needs == "" { + f.Needs = w.Needs + } + return f +} + +type plain struct{ Headline, Explanation, Resolved, Needs string } + +// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the +// controller's name for itself replaced by the self-check's where the words spoke of it. +var words = map[string]plain{ + "consumer-lost": {Headline: "A listener on the bus is gone", + Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", + Resolved: "The listener is back"}, + "consumer-wrong": {Headline: "A listener on the bus is set up wrong", + Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.", + Resolved: "The listener is set up right again"}, + "consumer-behind": {Headline: "Messages pile up for a listener", + Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " + + "restarts it if it stays.", + Resolved: "The listener caught up"}, + "stream-wrong": {Headline: "Part of the bus's storage is wrong", + Needs: "check the machine the bus runs on; the details say what is missing.", + Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", + Resolved: "The bus's storage is right again"}, + "bus-reconnects": {Headline: "A client keeps losing the bus", + Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + + "it reconnects, what it says and what it is asked waits.", + Resolved: "Resolved: the client stays connected"}, + "core-unhealthy": {Headline: "The bus is not healthy", + Needs: "check the machine the bus runs on; the details say what fails.", + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + + "calls may fail until it recovers.", + Resolved: "The bus is healthy again"}, + "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", + Needs: "restart the watchman if this stays, from the mesh MCP server.", + Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + + "signals are not noticed.", + Resolved: "The watchdogs run again"}, + "probe-failed": {Headline: "A check of the mesh could not run", + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + + "every few minutes.", + Resolved: "The check runs again"}, +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go new file mode 100644 index 00000000..44be64ce --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -0,0 +1,448 @@ +package main + +// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller, +// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home +// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the +// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the +// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and +// the bus step (its home is issue 1953's). + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat. +const ( + StreamsTool = "nats.nats_streams" + ClosedTool = "nats.nats_closed_connections" + SignalsVerb = "seat:mesh-watchdogs.signals" + notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; " + storeOnly = "held only in the controller's store today" + behindBound = 1000 + reconnectMax = 12 +) + +// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe +// that moves here is a row added, and the registry test holds it to ADR 0564's table. +func Registry() []Probe { + return []Probe{ + {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", + From: "issue 262", Kind: "resolver-wrong", + Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly}, + {ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " + + "is heard from", From: "issues 208, 218", Kind: "holder-silent", + Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " + + "records are " + storeOnly + ", and the watchman's presence has no read granted to this module"}, + {ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld", + Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly}, + {ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " + + "stream's head", From: "issues 248, 266", Kind: "consumer-wrong", + Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers}, + {ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208", + Kind: "stream-wrong", Run: probeStreams}, + {ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list", + From: "issue 238", Kind: "own-address-banned", + Deferred: "the machines' private addresses and endpoints: the overlay is " + storeOnly}, + {ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " + + "cleanup", From: "ADR 0230", Kind: "cleanup-waiting", + Deferred: "the retirement records of providers' consumers: " + storeOnly}, + {ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " + + "by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved", + Deferred: "the bindings of provisions to their consumers: " + storeOnly}, + {ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " + + "backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank", + Deferred: "the data items' measurements and their history: " + storeOnly}, + {ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " + + "bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects}, + {ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " + + "statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8", + Kind: "agent-can-become-root", + Deferred: "the node-engines' statements of the agent account's judgement: " + storeOnly}, + {ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " + + "proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free", + Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly}, + {ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " + + "filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501", + Kind: "bucket-or-log-filling", Raises: []string{"bucket-or-log-unread"}, + Deferred: "each bucket's and log's cap: the bus module's nats_buckets answers bytes and no cap, and the " + + "declared caps are " + storeOnly}, + {ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it", + From: "ADR 0376 §2", Kind: "grant-composes-nothing", + Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly}, + {ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals", + From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs}, + {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " + + "to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth}, + } +} + +// --- the bus's own state, as its module answers it --------------------------------------------------------------- + +// streamsAnswer is what nats_streams answers. +type streamsAnswer struct { + Detail []streamRow `json:"detail"` +} + +type streamRow struct { + Name string `json:"name"` + Subjects []string `json:"subjects"` + Retention string `json:"retention"` + MaxPerSubject int64 `json:"max_per_subject"` + Consumers []consumerRow `json:"consumer_detail"` +} + +type consumerRow struct { + Name string `json:"name"` + Kind string `json:"kind"` + Filters []string `json:"filters"` + Pending int64 `json:"pending"` + Unacked int64 `json:"unacknowledged"` + MaxDeliver int64 `json:"max_deliver"` +} + +// errNoStream is a stream the bus module says it does not hold. +var errNoStream = errors.New("no such stream") + +// streams asks the bus module for every stream, or for one with its consumers. +func (c *Checker) streams(stream string) ([]streamRow, error) { + args := map[string]any{} + if stream != "" { + args["stream"] = stream + } + raw, err := c.Ask(StreamsTool, args) + if err != nil { + if stream != "" && strings.Contains(err.Error(), "no stream called") { + return nil, errNoStream + } + return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) + } + var a streamsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err) + } + return a.Detail, nil +} + +// consumerWords is a consumer as the mesh says it, with its name. +func consumerWords(c buscompose.Consumer) string { + return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream) +} + +// expectedConsumers are the durable consumers the mesh defines, by stream. +func expectedConsumers() map[string][]buscompose.Consumer { + out := map[string][]buscompose.Consumer{} + for _, c := range buscompose.MeshConsumers() { + out[c.Stream] = append(out[c.Stream], c) + } + return out +} + +// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. +func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) { + var out []Observation + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if ctx.Err() != nil { + return nil, ctx.Err() + } + rows, err := c.streams(stream) + held := map[string]consumerRow{} + switch { + case errors.Is(err, errNoStream): + case err != nil: + return nil, err + default: + for _, r := range rows { + for _, cr := range r.Consumers { + held[cr.Name] = cr + } + } + } + for _, want := range byStream[stream] { + who := want.Stream + "." + want.Name + have, ok := held[want.Name] + if !ok { + out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), + Said: "no consumer " + want.Name + " on " + want.Stream}) + continue + } + if differs := consumerDiffers(want, have); differs != "" { + out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), + Said: differs}) + } + if want.Stream != "NODES" && have.Pending > behindBound { + // One look at a consumer a burst put behind is not one that stays behind: held for the next run, + // so healer H4 resets nothing on a single sample (novox/hq issue 277). + out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), + Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) + } + } + } + return sorted(out), nil +} + +// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its +// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is. +func consumerDiffers(want buscompose.Consumer, have consumerRow) string { + var differs []string + haveFilters := slices.Clone(have.Filters) + wantFilters := slices.Clone(want.Filters) + sort.Strings(haveFilters) + sort.Strings(wantFilters) + if !slices.Equal(haveFilters, wantFilters) { + differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters)) + } + if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) { + differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver)) + } + if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush { + differs = append(differs, "delivers by the other shape (push or pull) than defined") + } + return strings.Join(differs, "; ") +} + +// probeStreams is D7: every stream the mesh defines exists as defined. +func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { + rows, err := c.streams("") + if err != nil { + return nil, err + } + held := map[string]streamRow{} + for _, r := range rows { + held[r.Name] = r + } + var out []Observation + for _, s := range buscompose.MeshStreams() { + have, ok := held[s.Name] + if !ok { + out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), + Said: "no stream " + s.Name}) + continue + } + if differs := streamDiffers(s, have); differs != "" { + out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), + Said: differs}) + } + } + return sorted(out), nil +} + +// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its +// subjects, its retention and how many it keeps per subject. Empty when nothing is. +func streamDiffers(want buscompose.Stream, have streamRow) string { + var differs []string + haveSubjects := slices.Clone(have.Subjects) + wantSubjects := slices.Clone(want.Subjects) + sort.Strings(haveSubjects) + sort.Strings(wantSubjects) + if !slices.Equal(haveSubjects, wantSubjects) { + differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects)) + } + retention, perSubject := "limits", int64(want.MaxMsgsPerSubject) + switch want.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + perSubject = 1 + } + if have.Retention != "" && have.Retention != retention { + differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention)) + } + if perSubject != 0 && have.MaxPerSubject != perSubject { + differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject)) + } + return strings.Join(differs, "; ") +} + +// --- D15 ----------------------------------------------------------------------------------------------------- + +// closedAnswer is what nats_closed_connections answers. +type closedAnswer struct { + Hours float64 `json:"hours"` + Reaches bool `json:"reaches"` + Users []struct { + User string `json:"user"` + Closed int `json:"closed"` + Dropped int `json:"dropped"` + Names []struct { + Name string `json:"name"` + Dropped int `json:"dropped"` + Reasons map[string]int `json:"reasons"` + } `json:"names"` + } `json:"users"` +} + +// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. +func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { + raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) + if err != nil { + return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) + } + var read closedAnswer + if err := json.Unmarshal(raw, &read); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err) + } + hours := read.Hours + if hours <= 0 { + hours = 1 + } + var out []Observation + for _, u := range read.Users { + if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { + continue // refused before it logged in is nobody's reconnect loop + } + reasons := map[string]int{} + var names []string + for _, n := range u.Names { + if n.Dropped == 0 { + continue + } + names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped)) + for r, k := range n.Reasons { + if r != "Client Closed" { + reasons[r] += k + } + } + } + var why []string + for r, k := range reasons { + why = append(why, fmt.Sprintf("%s ×%d", r, k)) + } + sort.Strings(why) + partial := "" + if !read.Reaches { + partial = " (at least: the server's record of closed connections does not reach back the whole hour)" + } + who, machine := busUserWords(u.User) + out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, + Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ + "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), + Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, + hours, strings.Join(names, ", "), strings.Join(why, ", ")), + Headline: capital(who) + " keeps losing the bus", + Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+ + "again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)), + Resolved: "Resolved: " + who + " stays connected"}) + } + return sorted(out), nil +} + +// busUserWords is a bus user as the operator says it, and the machine it is on. +func busUserWords(user string) (string, string) { + switch { + case user == "controller": + return "the controller", "" + case strings.HasPrefix(user, "node."): + m := strings.TrimPrefix(user, "node.") + return "the node-engine on " + m, m + } + if m, module, ok := strings.Cut(user, "."); ok { + if module == "node-tools" { + return "the tool runner on " + m, m + } + return module + " on " + m, m + } + return "a client of the bus", "" +} + +// --- DW and H-bus -------------------------------------------------------------------------------------------- + +// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the +// watchman's pulse on `checked`, is the other half. +func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { + if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { + return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + + "late signal is being said", + Said: firstLine(err.Error())}}, nil + } + return nil, nil +} + +// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it +// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip, +// and no answer is no pass. +func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { + rows, err := c.streams("") + if err != nil { + return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) + } + held := map[string]bool{} + for _, r := range rows { + held[r.Name] = true + } + var problems []string + for _, s := range buscompose.MeshStreams() { + if !held[s.Name] { + problems = append(problems, "the stream "+s.Name+" is missing") + } + } + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if !held[stream] { + continue // said above + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + detail, err := c.streams(stream) + if err != nil && !errors.Is(err, errNoStream) { + return nil, err + } + have := map[string]bool{} + for _, r := range detail { + for _, cr := range r.Consumers { + have[cr.Name] = true + } + } + for _, want := range byStream[stream] { + if !have[want.Name] { + problems = append(problems, consumerWords(want)+" is missing") + } + } + } + if len(problems) == 0 { + return nil, nil + } + return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil +} + +// sorted is findings in a stated order, so two runs over one mesh say the same. +func sorted(out []Observation) []Observation { + sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) + return out +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go new file mode 100644 index 00000000..825416e0 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed. +type bus struct { + drop map[string]bool // stream or stream.consumer missing + change func(rows map[string]*streamRow) + closed string + signals error + asksFail error +} + +func (b bus) rows() map[string]*streamRow { + rows := map[string]*streamRow{} + for _, s := range buscompose.MeshStreams() { + if b.drop[s.Name] { + continue + } + retention := "limits" + per := int64(s.MaxMsgsPerSubject) + switch s.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + per = 1 + } + rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per} + } + for _, c := range buscompose.MeshConsumers() { + r := rows[c.Stream] + if r == nil || b.drop[c.Stream+"."+c.Name] { + continue + } + kind := "pull" + if c.Push || c.Queue != "" { + kind = "push" + } + r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)}) + } + if b.change != nil { + b.change(rows) + } + return rows +} + +func (b bus) ask(key string, body any) (json.RawMessage, error) { + if b.asksFail != nil { + return nil, b.asksFail + } + switch key { + case StreamsTool: + stream, _ := body.(map[string]any)["stream"].(string) + var detail []streamRow + for name, r := range b.rows() { + if stream == "" { + row := *r + row.Consumers = nil + detail = append(detail, row) + } else if name == stream { + detail = append(detail, *r) + } + } + if stream != "" && len(detail) == 0 { + return nil, errors.New("no stream called " + stream) + } + return json.Marshal(map[string]any{"detail": detail}) + case ClosedTool: + if b.closed == "" { + return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil + } + return json.RawMessage(b.closed), nil + case SignalsVerb: + if b.signals != nil { + return nil, b.signals + } + return json.RawMessage(`{"signals":[]}`), nil + } + return nil, errors.New("nats: no responders available for request") +} + +func keysOf(found []Observation) []string { + var out []string + for _, f := range found { + out = append(out, f.Key()) + } + return out +} + +// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2). +func TestTheBusProbes(t *testing.T) { + cases := []struct { + name string + probe func(context.Context, *Checker) ([]Observation, error) + bus bus + want []string + err string + }{ + {"D6 all as defined", probeConsumers, bus{}, nil, ""}, + {"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}}, + []string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, ""}, + {"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}}, + []string{"bus.CONTROL.controller.missing"}, ""}, + {"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) { + for i, c := range rows["EVENTS"].Consumers { + if c.Name == "controller" { + rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000 + } + } + }}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, ""}, + {"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer"}, + {"D7 all as defined", probeStreams, bus{}, nil, ""}, + {"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) { + rows["EVENTS"].Retention = "workqueue" + }}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, ""}, + {"D15 quiet", probeReconnects, bus{}, nil, ""}, + {"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[ + {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, + {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, + []string{"bus.g14.node-tools.bus-reconnects"}, ""}, + {"DW answers", probeWatchdogs, bus{}, nil, ""}, + {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, ""}, + {"H-bus healthy", probeBusHealth, bus{}, nil, ""}, + {"H-bus a stream and a consumer missing", probeBusHealth, + bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, ""}, + {"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed"}, + } + for _, tc := range cases { + c := &Checker{Ask: tc.bus.ask} + found, err := tc.probe(context.Background(), c) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Errorf("%s: error %v, want %q", tc.name, err, tc.err) + } + continue + } + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if got := keysOf(found); !slices.Equal(got, tc.want) { + t.Errorf("%s: found %v, want %v", tc.name, got, tc.want) + } + } +} + +// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what +// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). +func TestTheRegistryIsADR0564sTable(t *testing.T) { + moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", + "DW", "H-bus"} + var ids []string + for _, p := range Registry() { + ids = append(ids, p.ID) + if p.Kind == "" || p.Asserts == "" { + t.Errorf("%s has no kind or assertion", p.ID) + } + if (p.Run == nil) == (p.Deferred == "") { + t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID) + } + } + slices.Sort(ids) + slices.Sort(moved) + if !slices.Equal(ids, moved) { + t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved) + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/reads-outside b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside new file mode 100644 index 00000000..cb8f579b --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-self-check/module.json diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go new file mode 100644 index 00000000..53efb782 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -0,0 +1,394 @@ +package main + +// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five +// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket +// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the +// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the +// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is +// said in the verdict at once and written as a finding of its own when the next run cannot run it either. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-self-check" + Seat = "mesh-self-check" +) + +// The states it keeps, and the event that ends every run. +const ( + FindingsState = health.FindingsState + VerdictState = "verdict" + VerdictsState = "verdicts" + LastKey = "last" + CheckedEvent = health.EventChecked + // CheckedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent +) + +// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). +var ( + Every = 5 * time.Minute + FirstAfter = time.Minute + Within = 30 * time.Second +) + +// The verdicts a probe can have, as mesh-sdk's health names them. +const ( + VerdictPass = health.VerdictPassed + VerdictFail = health.VerdictFailed + VerdictFailedToRun = health.VerdictFailedToRun + VerdictDeferred = health.VerdictDeferred +) + +// sourceSelf is what writes a probe's failure to run: the self-check itself. +const sourceSelf = "doctor" + +// Store is the bus's buckets as this module reaches them, through its runtime. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits. +type Probe struct { + ID string + Asserts string + From string + Kind string + Raises []string + // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): + // answered deferred, never a pass. + Deferred string + Run func(ctx context.Context, c *Checker) ([]Observation, error) +} + +// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's +// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it. +type ( + ProbeVerdict = health.ProbeVerdict + Counts = health.CheckedCounts + Run = health.Checked +) + +// Checker is the self-check: the registry and what its probes reach. +type Checker struct { + Store Store + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) + Host string + Registry []Probe + + running sync.Mutex + mu sync.Mutex + n uint64 + last *Run + // seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is + // written only when two runs in a row see it, or while it is already written (novox/hq issue 277). + seen map[string]bool + // unsaid counts the runs whose `checked` could not be said since the start. + unsaid int +} + +// LastRun is the last run, nil before the first. +func (c *Checker) LastRun() *Run { + c.mu.Lock() + defer c.mu.Unlock() + return c.last +} + +type result struct { + found []Observation + err error + took time.Duration +} + +// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`. +// One run at a time: a person's run during a scheduled one waits for it. +func (c *Checker) RunOnce(ctx context.Context, why string) Run { + c.running.Lock() + defer c.running.Unlock() + c.mu.Lock() + c.n++ + n := c.n + c.mu.Unlock() + started := c.Now() + run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(), + IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why} + + results := make([]result, len(c.Registry)) + var wg sync.WaitGroup + for i, p := range c.Registry { + if p.Run == nil { + continue + } + wg.Add(1) + go func(i int, p Probe) { + defer wg.Done() + results[i] = c.probe(ctx, p) + }(i, p) + } + wg.Wait() + + written, readErr := c.written() + seen := map[string]bool{} + want := map[string]Observation{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Observation + for i, p := range c.Registry { + v := ProbeVerdict{ID: p.ID} + r := results[i] + switch { + case p.Run == nil: + v.Verdict = VerdictDeferred + v.Error = "deferred: " + p.Deferred + run.Counts.Deferred++ + case r.err != nil: + v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() + run.Counts.FailedToRun++ + blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, + Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), + Said: firstLine(r.err.Error())}) + default: + ran[p.ID] = true + for _, f := range r.found { + if f.Kind == "" { + f.Kind = p.Kind + } + f.Source = p.ID + f = worded(f) + key := f.Key() + seen[key] = true + if f.Confirm && !c.seenBefore(key) && !written.has(key) { + v.Unconfirmed = append(v.Unconfirmed, f.Summary) + continue + } + want[key] = f + v.Found = append(v.Found, f.Summary) + } + if len(v.Found) == 0 { + v.Verdict = VerdictPass + run.Counts.Passed++ + } else { + v.Verdict = VerdictFail + run.Counts.Failed++ + } + } + if p.Run != nil { + v.Took = r.took.Round(time.Millisecond).String() + } + run.Probes = append(run.Probes, v) + } + ran[sourceSelf] = true + for _, f := range blind { + f = worded(f) + key := f.Key() + seen[key] = true + if c.seenBefore(key) || written.has(key) { + want[key] = f + } + } + if readErr != nil { + // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. + c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) + } else if err := c.write(written, want, ran, run.Run); err != nil { + c.Logf("[%s] what this run found is not all written: %v", Module, err) + } + c.mu.Lock() + c.seen = seen + c.mu.Unlock() + + run.At = c.Now().UTC() + run.Took = run.At.Sub(started).Round(time.Millisecond).String() + c.mu.Lock() + run.Unsaid = c.unsaid + c.mu.Unlock() + c.keep(run) + return run +} + +// probe runs one probe within its bound; a panic or an overrun is an error, never a pass. +func (c *Checker) probe(ctx context.Context, p Probe) result { + probing, cancel := context.WithTimeout(ctx, Within) + defer cancel() + began := c.Now() + done := make(chan result, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- result{err: fmt.Errorf("the probe panicked: %v", r)} + } + }() + found, err := p.Run(probing, c) + done <- result{found: found, err: err} + }() + select { + case r := <-done: + r.took = c.Now().Sub(began) + return r + case <-probing.Done(): + return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)} + } +} + +func (c *Checker) seenBefore(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.seen[key] +} + +// writtenFindings are the findings in the bucket, by key. +type writtenFindings map[string]health.Finding + +func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } + +// written reads every finding this module has written. +func (c *Checker) written() (writtenFindings, error) { + keys, err := c.Store.Keys(FindingsState) + if err != nil { + return writtenFindings{}, err + } + out := writtenFindings{} + for _, k := range keys { + raw, found, err := c.Store.Get(FindingsState, k) + if err != nil { + return out, err + } + if !found { + continue + } + f, err := health.DecodeFinding(raw) + if err != nil { + return out, fmt.Errorf("the finding %s is not readable: %w", k, err) + } + out[k] = f + } + return out, nil +} + +// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding +// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). +func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error { + var errs []string + keys := make([]string, 0, len(want)) + for k := range want { + keys = append(keys, k) + } + sort.Strings(keys) + now := c.Now() + for _, k := range keys { + f := want[k].Finding(run, now) + if was, ok := written[k]; ok && says(was) == says(f) { + continue + } + raw, err := health.EncodeFinding(f) + if err == nil { + err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + for k, f := range written { + if _, still := want[k]; still || !ran[f.Source] { + continue + } + if err := c.Store.Delete(FindingsState, k); err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} + +// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in +// the log: the watchman's S10 says the self-check silent when `checked` stops. +func (c *Checker) keep(run Run) { + c.mu.Lock() + c.last = &run + c.mu.Unlock() + if err := c.Store.Put(VerdictState, LastKey, run); err != nil { + c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err) + } + if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil { + c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err) + } + if c.Publish == nil { + return + } + if err := c.Publish(CheckedSubject, run, run.Run); err != nil { + c.mu.Lock() + c.unsaid++ + c.mu.Unlock() + c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v", + Module, run.Run, err) + } +} + +// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314). +const overviewFindings = 20 + +// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all. +func Overview(run Run) Run { + probes := make([]ProbeVerdict, len(run.Probes)) + for i, p := range run.Probes { + total := len(p.Found) + len(p.Unconfirmed) + if total > overviewFindings { + p.FoundCount = len(p.Found) + if len(p.Found) > overviewFindings { + p.Found = p.Found[:overviewFindings:overviewFindings] + } + if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room { + p.Unconfirmed = p.Unconfirmed[:room:room] + } + p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all", + p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID) + } + probes[i] = p + } + run.Probes = probes + return run +} + +// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every. +func (c *Checker) Keep(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-time.After(FirstAfter): + } + tick := time.NewTicker(Every) + defer tick.Stop() + for { + c.RunOnce(ctx, "the schedule") + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go new file mode 100644 index 00000000..e79bb78f --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// memStore is the bus's buckets in memory, counting the writes. +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + puts map[string]int + keysErr error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) Put(state, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + s.kv[state][key] = raw + s.puts[state+"/"+key]++ + return nil +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.keysErr != nil && state == FindingsState { + return nil, s.keysErr + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) findingKeys() []string { + keys, _ := s.Keys(FindingsState) + return keys +} + +// checker is a self-check over memory, with a registry the test drives. +func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { + var said []string + now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) + c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Publish: func(subject string, body any, id string) error { + raw, _ := json.Marshal(body) + said = append(said, subject+" "+id+" "+string(raw)) + return nil + }} + return c, &said +} + +// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that +// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can +// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). +func TestARunWritesFindingsAsTheyChange(t *testing.T) { + var found []Observation + var fail error + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }} + deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} + missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + Summary: "the stream EVENTS is not on the bus"} + behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} + store := newStore() + c, said := checker(store, steady, deferred) + + steps := []struct { + name string + found []Observation + fail error + want []string // finding keys written after the run + verdict string + missingPs int // puts of the missing finding so far + }{ + {"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []Observation{missing, behind}, nil, + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"blind once: clears nothing, held", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, + {"blind twice: its own finding", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1}, + {"no longer seen: cleared", nil, nil, nil, VerdictPass, 1}, + } + for _, s := range steps { + found, fail = s.found, s.fail + run := c.RunOnce(context.Background(), "the test") + if got := store.findingKeys(); !slices.Equal(got, s.want) { + t.Errorf("%s: findings %v, want %v", s.name, got, s.want) + } + if run.Probes[0].Verdict != s.verdict { + t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict) + } + if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") { + t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1]) + } + if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs { + t.Errorf("%s: the unchanged finding was written %d times", s.name, n) + } + } + raw, _, _ := store.Get(FindingsState, "probe.P1.failed") + if raw != nil { + t.Errorf("the blind finding outlived the probe running again: %s", raw) + } + // Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body. + if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) { + t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps)) + } + // Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id. + if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") { + t.Fatalf("said %v", *said) + } + _, rest, _ := strings.Cut((*said)[0], " ") + _, rest, _ = strings.Cut(rest, " ") + var body map[string]any + _ = json.Unmarshal([]byte(rest), &body) + for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} { + if _, ok := body[field]; !ok { + t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body) + } + } + last, _, _ := store.Get(VerdictState, LastKey) + var r Run + _ = json.Unmarshal(last, &r) + if r.Counts.Passed != 1 || r.Counts.Deferred != 1 { + t.Errorf("the last verdict counts %+v", r.Counts) + } + // A finding carries the operator's words of its kind, the probe that found it and who observed it. + found, fail = []Observation{missing}, nil + c.RunOnce(context.Background(), "the test") + raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") + f, err := health.DecodeFinding(raw) + if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || + f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { + t.Errorf("the written finding is %+v", f) + } + // Blind to its own bucket, a run writes and clears nothing. + store.keysErr = errors.New("the bus is away") + found = nil + c.RunOnce(context.Background(), "the test") + store.keysErr = nil + if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) { + t.Errorf("a run blind to its findings changed them: %v", got) + } +} + +// A probe that panics or overruns its bound could not run: never a pass. +func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { + within := Within + Within = 50 * time.Millisecond + t.Cleanup(func() { Within = within }) + for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){ + "panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil }, + } { + c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) + c.Now = time.Now + r := c.RunOnce(context.Background(), "the test") + if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 { + t.Errorf("%s: %+v", name, r.Probes[0]) + } + } +} + +// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer +// every state the module keeps (ADR 0564 §12). +func TestTheVerbsAndReads(t *testing.T) { + store := newStore() + many := make([]Observation, 25) + for i := range many { + many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + } + c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", + Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }}) + if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { + t.Fatalf("before a run: %v", err) + } + if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil { + t.Fatal(err) + } + over, _ := c.Doctor(false, "", "") + if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" { + t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More) + } + whole, _ := c.Doctor(false, "d7", "") + if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 { + t.Errorf("one probe whole carries %d findings", len(v.Found)) + } + if _, err := c.Doctor(false, "D99", ""); err == nil { + t.Error("an unknown probe answered") + } + // A fresh checker over the same bucket answers the last verdict from it. + fresh, _ := checker(store) + if _, err := fresh.Doctor(false, "", ""); err != nil { + t.Errorf("the last verdict is not read from its bucket: %v", err) + } + fs, err := c.Findings("") + if err != nil || fs.(map[string]any)["count"] != 25 { + t.Errorf("findings answered %v, %v", fs, err) + } + if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") { + t.Errorf("one finding answered %v, %v", one, err) + } + vs, err := c.Verdicts("", 0) + if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 { + t.Errorf("verdicts answered %v, %v", vs, err) + } + if _, err := c.Verdicts("doctor-1-1", 0); err == nil { + t.Error("an unknown run answered") + } + if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" { + t.Errorf("probes answered %v", p) + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go new file mode 100644 index 00000000..1c1df753 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -0,0 +1,216 @@ +package main + +// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once, +// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the +// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"doctor", "probes"} + +// Tool names: the seat's verbs, and the module's own reads. +const ( + DoctorTool = Seat + ".doctor" + ProbesTool = Seat + ".probes" + FindingsTool = "self_check_findings" + VerdictsTool = "self_check_verdicts" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(c *Checker) []stdio.Tool { + return []stdio.Tool{ + {Name: DoctorTool, + Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " + + "failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " + + "every probe now and answers that run; probe= gives one probe's verdict whole, every finding.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "boolean", "description": "run every probe now"}, + "probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}), + Run: func(a map[string]any) (any, error) { + return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a)) + }}, + {Name: ProbesTool, + Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " + + "it raises, and for one deferred the facts it lacks on the bus.", + Run: func(map[string]any) (any, error) { return c.Probes(), nil }}, + {Name: FindingsTool, + Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " + + "finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}), + Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }}, + {Name: VerdictsTool, + Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " + + "with its counts; or one run whole by its id. The last run is the doctor verb's.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"}, + "limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}), + Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }}, + } +} + +// Doctor is the verb: the last verdict, a run now, or one probe of either. +func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) { + var r Run + switch { + case run: + r = c.RunOnce(context.Background(), why) + case c.LastRun() != nil: + r = *c.LastRun() + default: + raw, found, err := c.Store.Get(VerdictState, LastKey) + if err != nil { + return nil, fmt.Errorf("the last verdict cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+ + "every %s — run=true runs it now", FirstAfter, Every) + } + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("the last verdict is not readable: %w", err) + } + } + if probe != "" { + one, ok := oneProbe(r, probe) + if !ok { + return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool) + } + r = one + } else { + r = Overview(r) + } + return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(), + "note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " + + "longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil +} + +func oneProbe(r Run, id string) (Run, bool) { + for _, p := range r.Probes { + if strings.EqualFold(p.ID, id) { + r.Probes = []ProbeVerdict{p} + return r, true + } + } + return Run{}, false +} + +// Probes is the registry as the verb answers it. +func (c *Checker) Probes() map[string]any { + var out []map[string]any + for _, p := range c.Registry { + row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind} + if len(p.Raises) > 0 { + row["raises"] = p.Raises + } + if p.Run == nil { + row["deferred"] = p.Deferred + } + out = append(out, row) + } + return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()} +} + +// Findings are the findings written now, or one whole. +func (c *Checker) Findings(key string) (any, error) { + if key != "" { + raw, found, err := c.Store.Get(FindingsState, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no finding is written under %s now", key) + } + return json.RawMessage(raw), nil + } + written, err := c.written() + if err != nil { + return nil, fmt.Errorf("the findings cannot be read: %w", err) + } + rows := []map[string]any{} + for k, f := range written { + rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source, + "summary": f.Summary}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) }) + return map[string]any{"findings": rows, "count": len(rows)}, nil +} + +// Verdicts are the week's runs newest first, or one whole. +func (c *Checker) Verdicts(run string, limit int) (any, error) { + if run != "" { + raw, found, err := c.Store.Get(VerdictsState, run) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run) + } + return json.RawMessage(raw), nil + } + keys, err := c.Store.Keys(VerdictsState) + if err != nil { + return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err) + } + sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) }) + total := len(keys) + if limit > 0 && len(keys) > limit { + keys = keys[:limit] + } + rows := []map[string]any{} + for _, k := range keys { + raw, found, err := c.Store.Get(VerdictsState, k) + if err != nil || !found { + continue + } + var r Run + if json.Unmarshal(raw, &r) == nil { + rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts}) + } + } + return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil +} + +// runOrder orders run ids `doctor--` by time, then number. +func runOrder(id string) string { + parts := strings.Split(id, "-") + if len(parts) != 3 { + return id + } + return fmt.Sprintf("%020s-%020s", parts[1], parts[2]) +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as the seat's "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go new file mode 100644 index 00000000..54ae1679 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -0,0 +1,298 @@ +//go:build integration + +package main + +// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real +// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from +// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus, +// writes them as findings into its bucket on the bus, keeps the verdict, and says checked. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + } + _ = json.Unmarshal(m.Params, &ask) + if ask.Key != StreamsTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + // The bus module's answer, from the server's own streams. + var detail []map[string]any + for name := range p.js.StreamNames(p.ctx).Name() { + detail = append(detail, map[string]any{"name": name}) + } + p.answer(m.ID, map[string]any{"detail": detail}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus. +func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the self-check does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{FindingsState, VerdictState, VerdictsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil { + t.Fatal(err) + } + } + checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(DoctorTool, map[string]any{"run": true}) + if err != nil { + t.Fatal(err) + } + var answer struct { + Run Run `json:"run"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + t.Fatal(err) + } + verdicts := map[string]string{} + for _, v := range answer.Run.Probes { + verdicts[v.ID] = v.Verdict + } + if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun || + verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") { + t.Errorf("the run answered %+v", answer.Run) + } + msg, err := checked.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("checked was not said: %v", err) + } + var said Run + if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run { + t.Errorf("checked said %s", msg.Data) + } + findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState) + e, err := findings.Get(ctx, "bus.EVENTS.stream-missing") + if err != nil { + t.Fatalf("the missing stream is not a finding on the bus: %v", err) + } + f, err := health.DecodeFinding(e.Value()) + if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + t.Errorf("the finding is %s", e.Value()) + } + verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) + if _, err := verdict.Get(ctx, LastKey); err != nil { + t.Errorf("the last verdict is not kept: %v", err) + } + if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) { + t.Errorf("the week's verdicts answered %s, %v", raw, err) + } +} diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod new file mode 100644 index 00000000..fd63c2a8 --- /dev/null +++ b/modules/mesh-self-check/go.mod @@ -0,0 +1,22 @@ +module mesh-self-check + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum new file mode 100644 index 00000000..a0208db4 --- /dev/null +++ b/modules/mesh-self-check/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json new file mode 100644 index 00000000..829af53e --- /dev/null +++ b/modules/mesh-self-check/module.json @@ -0,0 +1,64 @@ +{ + "module": "mesh-self-check", + "version": "1", + "runs-as": "mesh-self-check", + "claims": [ + { + "name": "mesh-self-check", + "scope": "mesh", + "serves": [ + "doctor", + "probes" + ] + } + ], + "invokes": [ + "nats.nats_streams", + "nats.nats_closed_connections", + "seat:mesh-watchdogs.signals" + ], + "state": [ + "findings", + "verdict", + { + "name": "verdicts", + "ttl-seconds": 604800 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-self-check", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-self-check", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-self-check" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-self-check", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-self-check", + "binary": "mesh-self-check", + "loads": [ + "mesh-self-check" + ] + } + ] + } +}