From d7247eae19e07eedefc697d008e7b3dc24e156cb Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:37:43 +0100 Subject: [PATCH 01/20] Move the self-check out of the controller into mesh-self-check, so its runs and findings no longer need the controller's lease (issue 1952) The probes whose facts are on the bus run with their assertions unchanged (D6, D7, D15, DW, H-bus); the rest answer deferred, naming what they lack, never a pass. Findings go to its own bucket for the condition keeper; each run is kept a week and said as checked. The finding layout is a copy of mesh-sdk's S3 until that lands. --- modules/mesh-self-check/.gitignore | 2 + modules/mesh-self-check/README.md | 33 ++ .../cmd/mesh-self-check/finding.go | 149 ++++++ .../cmd/mesh-self-check/main.go | 57 +++ .../cmd/mesh-self-check/manifest_test.go | 100 ++++ .../cmd/mesh-self-check/probes.go | 448 ++++++++++++++++++ .../cmd/mesh-self-check/probes_test.go | 176 +++++++ .../cmd/mesh-self-check/reads-outside | 2 + .../cmd/mesh-self-check/selfcheck.go | 412 ++++++++++++++++ .../cmd/mesh-self-check/selfcheck_test.go | 250 ++++++++++ .../cmd/mesh-self-check/tools.go | 210 ++++++++ .../mesh-self-check/whole_integration_test.go | 290 ++++++++++++ modules/mesh-self-check/go.mod | 22 + modules/mesh-self-check/go.sum | 27 ++ modules/mesh-self-check/module.json | 67 +++ 15 files changed, 2245 insertions(+) create mode 100644 modules/mesh-self-check/.gitignore create mode 100644 modules/mesh-self-check/README.md create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/finding.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/main.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_test.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/reads-outside create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/tools.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go create mode 100644 modules/mesh-self-check/go.mod create mode 100644 modules/mesh-self-check/go.sum create mode 100644 modules/mesh-self-check/module.json diff --git a/modules/mesh-self-check/.gitignore b/modules/mesh-self-check/.gitignore new file mode 100644 index 00000000..2c7bac8b --- /dev/null +++ b/modules/mesh-self-check/.gitignore @@ -0,0 +1,2 @@ +/mesh-self-check +/cmd/mesh-self-check/mesh-self-check diff --git a/modules/mesh-self-check/README.md b/modules/mesh-self-check/README.md new file mode 100644 index 00000000..9411474c --- /dev/null +++ b/modules/mesh-self-check/README.md @@ -0,0 +1,33 @@ +# mesh-self-check + +Holds the seat `mesh-self-check` (novox/hq ADR 0564 §6): the mesh's self-check, moved out of the controller. Go, one +binary (`cmd/mesh-self-check`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes its buckets. + +## What it does + +Every five minutes (the first run a minute after it starts) it runs its registry of probes, each within thirty +seconds. A probe is one live invariant of a design, with its assertion as the controller had it, reading its facts on +the bus (the bus module's tools for the bus's own state). A probe whose facts have no home on the bus yet answers +`deferred`, naming them: never a pass. A probe that cannot run is said in the verdict at once, and written as a finding +of its own (`probe..failed`) when the next run cannot run it either. + +What a probe finds is written as a finding, one key `..` per fault, only when what it says changes, and +deleted when a run no longer sees it. The condition keeper (`mesh-conditions`) raises and clears conditions from them. +A finding one look can be wrong about (a consumer behind after a burst) is written when two runs in a row see it. + +Every run ends with the event `checked`, the body of the controller's `doctor-heartbeat`, which the watchman (S10) and +mesh-watcher on another machine hear: a self-check that stops checking is said through a path that does not pass +through it. + +## What it keeps + +- `mesh-self-check_findings`: what it finds wrong now. +- `mesh-self-check_verdict`: the key `last`, the last run whole. +- `mesh-self-check_verdicts`: every run (each probe's findings at most twenty), kept a week by the bucket's age. + +## Its verbs and tools + +- `mesh-self-check.doctor`: the last verdict; `run=true` runs now; `probe=` one probe whole. +- `mesh-self-check.probes`: the registry, with what each deferred probe lacks. +- `self_check_findings`, `self_check_verdicts`: the reads of its buckets. diff --git a/modules/mesh-self-check/cmd/mesh-self-check/finding.go b/modules/mesh-self-check/cmd/mesh-self-check/finding.go new file mode 100644 index 00000000..9a20352e --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/finding.go @@ -0,0 +1,149 @@ +package main + +// A finding: what an observer writes for the condition keeper (novox/hq ADR 0564 §2). **A copy of the contract +// mesh-sdk holds (issue 1952, S3)**: its layout is the controller's observation (mesh-controller +// internal/conditions/condition.go), its key the controller's key rule, so a finding written here and the condition +// the controller raised for the same fault share one key. When mesh-sdk's go/health carries the finding, this file +// is replaced by it and nothing else changes. + +import ( + "encoding/json" + "regexp" + "strings" +) + +// The two severities a condition has (to-be 45 §2). +const ( + Urgent = "urgent" + Warning = "warning" +) + +// The scopes a key starts with, as the controller names them. +const ( + ScopeBus = "bus" + ScopeCore = "core" + ScopeProbe = "probe" + ScopeMesh = "mesh" +) + +// Subject is what a finding is about: its scope, its id within the scope, the machine it concerns. +type Subject struct { + Scope string `json:"scope"` + ID string `json:"id"` + Machine string `json:"machine,omitempty"` + Also []string `json:"also,omitempty"` +} + +// Finding is one thing an observer sees wrong now, kept under its key in the observer's bucket `findings` and +// deleted when it is no longer seen. Token and Confirm are the observer's own and never written. +type Finding struct { + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity string `json:"severity"` + // Summary is one line in the mesh's words; Said the evidence (an address or a name stays here, never in the + // summary, ADR 0234 §6). + Summary string `json:"summary"` + Said string `json:"said,omitempty"` + // Headline, Explanation, Resolved and Needs are what the operator reads (ADR 0253). + Headline string `json:"headline"` + Explanation string `json:"explanation"` + Resolved string `json:"resolved"` + Needs string `json:"needs,omitempty"` + // Source is what found it: a probe's id, or the observer itself for its own blindness. + Source string `json:"source"` + // Observer is the module that wrote it. + Observer string `json:"observer"` + + Token string `json:"-"` + Confirm bool `json:"-"` +} + +// Key is where the finding is kept: `..`, the controller's key for the same condition. +func (f Finding) Key() string { + token := f.Token + if token == "" { + token = f.Kind + } + return Key(f.Subject.Scope, f.Subject.ID, token) +} + +// Says is what the finding says, as written: two findings that say the same are one write (ADR 0289 §1). +func (f Finding) Says() string { + raw, _ := json.Marshal(f) + return string(raw) +} + +var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) + +// Key composes a key from its parts, each token made safe for the bus, as the controller's conditions.Key does. +func Key(scope, id, token string) string { + var parts []string + for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { + p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") + if p == "" { + p = "_" + } + parts = append(parts, p) + } + return strings.Join(parts, ".") +} + +// worded gives a finding the operator's words registered for its kind, where it says none of its own. +func worded(f Finding) Finding { + w, ok := words[f.Kind] + if !ok { + return f + } + if f.Headline == "" { + f.Headline = w.Headline + } + if f.Explanation == "" { + f.Explanation = w.Explanation + } + if f.Resolved == "" { + f.Resolved = w.Resolved + } + if f.Needs == "" { + f.Needs = w.Needs + } + return f +} + +type plain struct{ Headline, Explanation, Resolved, Needs string } + +// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the +// controller's name for itself replaced by the self-check's where the words spoke of it. +var words = map[string]plain{ + "consumer-lost": {Headline: "A listener on the bus is gone", + Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", + Resolved: "The listener is back"}, + "consumer-wrong": {Headline: "A listener on the bus is set up wrong", + Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.", + Resolved: "The listener is set up right again"}, + "consumer-behind": {Headline: "Messages pile up for a listener", + Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " + + "restarts it if it stays.", + Resolved: "The listener caught up"}, + "stream-wrong": {Headline: "Part of the bus's storage is wrong", + Needs: "check the machine the bus runs on; the details say what is missing.", + Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", + Resolved: "The bus's storage is right again"}, + "bus-reconnects": {Headline: "A client keeps losing the bus", + Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + + "it reconnects, what it says and what it is asked waits.", + Resolved: "Resolved: the client stays connected"}, + "core-unhealthy": {Headline: "The bus is not healthy", + Needs: "check the machine the bus runs on; the details say what fails.", + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + + "calls may fail until it recovers.", + Resolved: "The bus is healthy again"}, + "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", + Needs: "restart the watchman if this stays, from the mesh MCP server.", + Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + + "signals are not noticed.", + Resolved: "The watchdogs run again"}, + "probe-failed": {Headline: "A check of the mesh could not run", + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + + "every few minutes.", + Resolved: "The check runs again"}, +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go new file mode 100644 index 00000000..c423e855 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -0,0 +1,57 @@ +// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds +// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts` +// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is +// the MCP channel; what this module says, it says on stderr. +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// NewChecker is the self-check over the runtime. +func NewChecker() *Checker { + return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Emit: func(event string, body any) error { return stdio.Emit(event, body) }} +} + +func main() { + c := NewChecker() + go func() { + <-stdio.Served() + c.Keep(context.Background()) + }() + if err := stdio.Serve("", Tools(c)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go new file mode 100644 index 00000000..10f33dc9 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves +// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts, +// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by +// itself a tool that starts it by hand (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + m := readManifest(t) + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + if !slices.Equal(m.Emits, []string{CheckedEvent}) { + t.Errorf("emits %v", m.Emits) + } + if !slices.Equal(m.Invokes, []string{StreamsTool, ClosedTool, SignalsVerb}) { + t.Errorf("invokes %v, want what the probes ask", m.Invokes) + } + if len(m.Reads) != 0 { + t.Errorf("reads %v: the self-check reads its facts through the bus module's tools", m.Reads) + } + states := map[string]int{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + TTL int `json:"ttl-seconds"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.TTL + } + if states[VerdictsState] != 7*24*60*60 { + t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState]) + } + // ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool, + "the run": DoctorTool + " run=true"} + tools := map[string]bool{} + for _, tool := range Tools(&Checker{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + name, _, _ := cutSpace(tool) + if !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} + +func cutSpace(s string) (string, string, bool) { + for i, r := range s { + if r == ' ' { + return s[:i], s[i+1:], true + } + } + return s, "", false +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go new file mode 100644 index 00000000..9f4e9882 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -0,0 +1,448 @@ +package main + +// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller, +// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home +// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the +// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the +// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and +// the bus step (its home is issue 1953's). + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat. +const ( + StreamsTool = "nats.nats_streams" + ClosedTool = "nats.nats_closed_connections" + SignalsVerb = "seat:mesh-watchdogs.signals" + notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; " + storeOnly = "held only in the controller's store today" + behindBound = 1000 + reconnectMax = 12 +) + +// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe +// that moves here is a row added, and the registry test holds it to ADR 0564's table. +func Registry() []Probe { + return []Probe{ + {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", + From: "issue 262", Kind: "resolver-wrong", + Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly}, + {ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " + + "is heard from", From: "issues 208, 218", Kind: "holder-silent", + Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " + + "records are " + storeOnly + ", and the watchman's presence has no read granted to this module"}, + {ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld", + Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly}, + {ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " + + "stream's head", From: "issues 248, 266", Kind: "consumer-wrong", + Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers}, + {ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208", + Kind: "stream-wrong", Run: probeStreams}, + {ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list", + From: "issue 238", Kind: "own-address-banned", + Deferred: "the machines' private addresses and endpoints: the overlay is " + storeOnly}, + {ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " + + "cleanup", From: "ADR 0230", Kind: "cleanup-waiting", + Deferred: "the retirement records of providers' consumers: " + storeOnly}, + {ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " + + "by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved", + Deferred: "the bindings of provisions to their consumers: " + storeOnly}, + {ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " + + "backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank", + Deferred: "the data items' measurements and their history: " + storeOnly}, + {ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " + + "bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects}, + {ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " + + "statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8", + Kind: "agent-can-become-root", + Deferred: "the node-engines' statements of the agent account's judgement: " + storeOnly}, + {ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " + + "proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free", + Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly}, + {ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " + + "filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501", + Kind: "bucket-or-log-filling", Raises: []string{"bucket-or-log-unread"}, + Deferred: "each bucket's and log's cap: the bus module's nats_buckets answers bytes and no cap, and the " + + "declared caps are " + storeOnly}, + {ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it", + From: "ADR 0376 §2", Kind: "grant-composes-nothing", + Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly}, + {ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals", + From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs}, + {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " + + "to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth}, + } +} + +// --- the bus's own state, as its module answers it --------------------------------------------------------------- + +// streamsAnswer is what nats_streams answers. +type streamsAnswer struct { + Detail []streamRow `json:"detail"` +} + +type streamRow struct { + Name string `json:"name"` + Subjects []string `json:"subjects"` + Retention string `json:"retention"` + MaxPerSubject int64 `json:"max_per_subject"` + Consumers []consumerRow `json:"consumer_detail"` +} + +type consumerRow struct { + Name string `json:"name"` + Kind string `json:"kind"` + Filters []string `json:"filters"` + Pending int64 `json:"pending"` + Unacked int64 `json:"unacknowledged"` + MaxDeliver int64 `json:"max_deliver"` +} + +// errNoStream is a stream the bus module says it does not hold. +var errNoStream = errors.New("no such stream") + +// streams asks the bus module for every stream, or for one with its consumers. +func (c *Checker) streams(stream string) ([]streamRow, error) { + args := map[string]any{} + if stream != "" { + args["stream"] = stream + } + raw, err := c.Ask(StreamsTool, args) + if err != nil { + if stream != "" && strings.Contains(err.Error(), "no stream called") { + return nil, errNoStream + } + return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) + } + var a streamsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err) + } + return a.Detail, nil +} + +// consumerWords is a consumer as the mesh says it, with its name. +func consumerWords(c buscompose.Consumer) string { + return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream) +} + +// expectedConsumers are the durable consumers the mesh defines, by stream. +func expectedConsumers() map[string][]buscompose.Consumer { + out := map[string][]buscompose.Consumer{} + for _, c := range buscompose.MeshConsumers() { + out[c.Stream] = append(out[c.Stream], c) + } + return out +} + +// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. +func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) { + var out []Finding + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if ctx.Err() != nil { + return nil, ctx.Err() + } + rows, err := c.streams(stream) + held := map[string]consumerRow{} + switch { + case errors.Is(err, errNoStream): + case err != nil: + return nil, err + default: + for _, r := range rows { + for _, cr := range r.Consumers { + held[cr.Name] = cr + } + } + } + for _, want := range byStream[stream] { + who := want.Stream + "." + want.Name + have, ok := held[want.Name] + if !ok { + out = append(out, Finding{Kind: "consumer-lost", Token: "missing", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), + Said: "no consumer " + want.Name + " on " + want.Stream}) + continue + } + if differs := consumerDiffers(want, have); differs != "" { + out = append(out, Finding{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), + Said: differs}) + } + if want.Stream != "NODES" && have.Pending > behindBound { + // One look at a consumer a burst put behind is not one that stays behind: held for the next run, + // so healer H4 resets nothing on a single sample (novox/hq issue 277). + out = append(out, Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), + Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) + } + } + } + return sorted(out), nil +} + +// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its +// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is. +func consumerDiffers(want buscompose.Consumer, have consumerRow) string { + var differs []string + haveFilters := slices.Clone(have.Filters) + wantFilters := slices.Clone(want.Filters) + sort.Strings(haveFilters) + sort.Strings(wantFilters) + if !slices.Equal(haveFilters, wantFilters) { + differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters)) + } + if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) { + differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver)) + } + if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush { + differs = append(differs, "delivers by the other shape (push or pull) than defined") + } + return strings.Join(differs, "; ") +} + +// probeStreams is D7: every stream the mesh defines exists as defined. +func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) { + rows, err := c.streams("") + if err != nil { + return nil, err + } + held := map[string]streamRow{} + for _, r := range rows { + held[r.Name] = r + } + var out []Finding + for _, s := range buscompose.MeshStreams() { + have, ok := held[s.Name] + if !ok { + out = append(out, Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), + Said: "no stream " + s.Name}) + continue + } + if differs := streamDiffers(s, have); differs != "" { + out = append(out, Finding{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), + Said: differs}) + } + } + return sorted(out), nil +} + +// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its +// subjects, its retention and how many it keeps per subject. Empty when nothing is. +func streamDiffers(want buscompose.Stream, have streamRow) string { + var differs []string + haveSubjects := slices.Clone(have.Subjects) + wantSubjects := slices.Clone(want.Subjects) + sort.Strings(haveSubjects) + sort.Strings(wantSubjects) + if !slices.Equal(haveSubjects, wantSubjects) { + differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects)) + } + retention, perSubject := "limits", int64(want.MaxMsgsPerSubject) + switch want.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + perSubject = 1 + } + if have.Retention != "" && have.Retention != retention { + differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention)) + } + if perSubject != 0 && have.MaxPerSubject != perSubject { + differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject)) + } + return strings.Join(differs, "; ") +} + +// --- D15 ----------------------------------------------------------------------------------------------------- + +// closedAnswer is what nats_closed_connections answers. +type closedAnswer struct { + Hours float64 `json:"hours"` + Reaches bool `json:"reaches"` + Users []struct { + User string `json:"user"` + Closed int `json:"closed"` + Dropped int `json:"dropped"` + Names []struct { + Name string `json:"name"` + Dropped int `json:"dropped"` + Reasons map[string]int `json:"reasons"` + } `json:"names"` + } `json:"users"` +} + +// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. +func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { + raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) + if err != nil { + return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) + } + var read closedAnswer + if err := json.Unmarshal(raw, &read); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err) + } + hours := read.Hours + if hours <= 0 { + hours = 1 + } + var out []Finding + for _, u := range read.Users { + if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { + continue // refused before it logged in is nobody's reconnect loop + } + reasons := map[string]int{} + var names []string + for _, n := range u.Names { + if n.Dropped == 0 { + continue + } + names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped)) + for r, k := range n.Reasons { + if r != "Client Closed" { + reasons[r] += k + } + } + } + var why []string + for r, k := range reasons { + why = append(why, fmt.Sprintf("%s ×%d", r, k)) + } + sort.Strings(why) + partial := "" + if !read.Reaches { + partial = " (at least: the server's record of closed connections does not reach back the whole hour)" + } + who, machine := busUserWords(u.User) + out = append(out, Finding{Kind: "bus-reconnects", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, + Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ + "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), + Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, + hours, strings.Join(names, ", "), strings.Join(why, ", ")), + Headline: capital(who) + " keeps losing the bus", + Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+ + "again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)), + Resolved: "Resolved: " + who + " stays connected"}) + } + return sorted(out), nil +} + +// busUserWords is a bus user as the operator says it, and the machine it is on. +func busUserWords(user string) (string, string) { + switch { + case user == "controller": + return "the controller", "" + case strings.HasPrefix(user, "node."): + m := strings.TrimPrefix(user, "node.") + return "the node-engine on " + m, m + } + if m, module, ok := strings.Cut(user, "."); ok { + if module == "node-tools" { + return "the tool runner on " + m, m + } + return module + " on " + m, m + } + return "a client of the bus", "" +} + +// --- DW and H-bus -------------------------------------------------------------------------------------------- + +// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the +// watchman's pulse on `checked`, is the other half. +func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) { + if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { + return []Finding{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + + "late signal is being said", + Said: firstLine(err.Error())}}, nil + } + return nil, nil +} + +// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it +// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip, +// and no answer is no pass. +func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) { + rows, err := c.streams("") + if err != nil { + return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) + } + held := map[string]bool{} + for _, r := range rows { + held[r.Name] = true + } + var problems []string + for _, s := range buscompose.MeshStreams() { + if !held[s.Name] { + problems = append(problems, "the stream "+s.Name+" is missing") + } + } + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if !held[stream] { + continue // said above + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + detail, err := c.streams(stream) + if err != nil && !errors.Is(err, errNoStream) { + return nil, err + } + have := map[string]bool{} + for _, r := range detail { + for _, cr := range r.Consumers { + have[cr.Name] = true + } + } + for _, want := range byStream[stream] { + if !have[want.Name] { + problems = append(problems, consumerWords(want)+" is missing") + } + } + } + if len(problems) == 0 { + return nil, nil + } + return []Finding{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil +} + +// sorted is findings in a stated order, so two runs over one mesh say the same. +func sorted(out []Finding) []Finding { + sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) + return out +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go new file mode 100644 index 00000000..7b918757 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed. +type bus struct { + drop map[string]bool // stream or stream.consumer missing + change func(rows map[string]*streamRow) + closed string + signals error + asksFail error +} + +func (b bus) rows() map[string]*streamRow { + rows := map[string]*streamRow{} + for _, s := range buscompose.MeshStreams() { + if b.drop[s.Name] { + continue + } + retention := "limits" + per := int64(s.MaxMsgsPerSubject) + switch s.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + per = 1 + } + rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per} + } + for _, c := range buscompose.MeshConsumers() { + r := rows[c.Stream] + if r == nil || b.drop[c.Stream+"."+c.Name] { + continue + } + kind := "pull" + if c.Push || c.Queue != "" { + kind = "push" + } + r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)}) + } + if b.change != nil { + b.change(rows) + } + return rows +} + +func (b bus) ask(key string, body any) (json.RawMessage, error) { + if b.asksFail != nil { + return nil, b.asksFail + } + switch key { + case StreamsTool: + stream, _ := body.(map[string]any)["stream"].(string) + var detail []streamRow + for name, r := range b.rows() { + if stream == "" { + row := *r + row.Consumers = nil + detail = append(detail, row) + } else if name == stream { + detail = append(detail, *r) + } + } + if stream != "" && len(detail) == 0 { + return nil, errors.New("no stream called " + stream) + } + return json.Marshal(map[string]any{"detail": detail}) + case ClosedTool: + if b.closed == "" { + return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil + } + return json.RawMessage(b.closed), nil + case SignalsVerb: + if b.signals != nil { + return nil, b.signals + } + return json.RawMessage(`{"signals":[]}`), nil + } + return nil, errors.New("nats: no responders available for request") +} + +func keysOf(found []Finding) []string { + var out []string + for _, f := range found { + out = append(out, f.Key()) + } + return out +} + +// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2). +func TestTheBusProbes(t *testing.T) { + cases := []struct { + name string + probe func(context.Context, *Checker) ([]Finding, error) + bus bus + want []string + err string + }{ + {"D6 all as defined", probeConsumers, bus{}, nil, ""}, + {"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}}, + []string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, ""}, + {"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}}, + []string{"bus.CONTROL.controller.missing"}, ""}, + {"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) { + for i, c := range rows["EVENTS"].Consumers { + if c.Name == "controller" { + rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000 + } + } + }}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, ""}, + {"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer"}, + {"D7 all as defined", probeStreams, bus{}, nil, ""}, + {"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) { + rows["EVENTS"].Retention = "workqueue" + }}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, ""}, + {"D15 quiet", probeReconnects, bus{}, nil, ""}, + {"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[ + {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, + {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, + []string{"bus.g14.node-tools.bus-reconnects"}, ""}, + {"DW answers", probeWatchdogs, bus{}, nil, ""}, + {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, ""}, + {"H-bus healthy", probeBusHealth, bus{}, nil, ""}, + {"H-bus a stream and a consumer missing", probeBusHealth, + bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, ""}, + {"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed"}, + } + for _, tc := range cases { + c := &Checker{Ask: tc.bus.ask} + found, err := tc.probe(context.Background(), c) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Errorf("%s: error %v, want %q", tc.name, err, tc.err) + } + continue + } + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if got := keysOf(found); !slices.Equal(got, tc.want) { + t.Errorf("%s: found %v, want %v", tc.name, got, tc.want) + } + } +} + +// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what +// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). +func TestTheRegistryIsADR0564sTable(t *testing.T) { + moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", + "DW", "H-bus"} + var ids []string + for _, p := range Registry() { + ids = append(ids, p.ID) + if p.Kind == "" || p.Asserts == "" { + t.Errorf("%s has no kind or assertion", p.ID) + } + if (p.Run == nil) == (p.Deferred == "") { + t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID) + } + } + slices.Sort(ids) + slices.Sort(moved) + if !slices.Equal(ids, moved) { + t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved) + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/reads-outside b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside new file mode 100644 index 00000000..cb8f579b --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-self-check/module.json diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go new file mode 100644 index 00000000..aa670108 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -0,0 +1,412 @@ +package main + +// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five +// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket +// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the +// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the +// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is +// said in the verdict at once and written as a finding of its own when the next run cannot run it either. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-self-check" + Seat = "mesh-self-check" +) + +// The states it keeps, and the event that ends every run. +const ( + FindingsState = "findings" + VerdictState = "verdict" + VerdictsState = "verdicts" + LastKey = "last" + CheckedEvent = "checked" +) + +// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). +var ( + Every = 5 * time.Minute + FirstAfter = time.Minute + Within = 30 * time.Second +) + +// The verdicts a probe can have. +const ( + VerdictPass = "pass" + VerdictFail = "fail" + VerdictFailedToRun = "failed-to-run" + VerdictDeferred = "deferred" +) + +// sourceSelf is what writes a probe's failure to run: the self-check itself. +const sourceSelf = "doctor" + +// Store is the bus's buckets as this module reaches them, through its runtime. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits. +type Probe struct { + ID string + Asserts string + From string + Kind string + Raises []string + // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): + // answered deferred, never a pass. + Deferred string + Run func(ctx context.Context, c *Checker) ([]Finding, error) +} + +// ProbeVerdict is one probe's outcome in a run, as the controller's doctor says it. +type ProbeVerdict struct { + ID string `json:"id"` + Verdict string `json:"verdict"` + Found []string `json:"found,omitempty"` + Unconfirmed []string `json:"unconfirmed,omitempty"` + Error string `json:"error,omitempty"` + Took string `json:"took,omitempty"` + FoundCount int `json:"found-count,omitempty"` + More string `json:"more,omitempty"` +} + +// Counts are a run's verdicts, counted. +type Counts struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + FailedToRun int `json:"failed-to-run"` + Deferred int `json:"deferred"` +} + +// Run is one run of the registry, and the body of `checked`: the controller's doctor-heartbeat body unchanged +// (ADR 0564 §6), Controller naming the machine that ran it. +type Run struct { + Run string `json:"run"` + At time.Time `json:"at"` + Started time.Time `json:"started"` + Took string `json:"took"` + IntervalSeconds int `json:"interval-seconds"` + Counts Counts `json:"counts"` + Probes []ProbeVerdict `json:"probes"` + Controller string `json:"controller"` + Why string `json:"why"` + Unsaid int `json:"unsaid,omitempty"` +} + +// Checker is the self-check: the registry and what its probes reach. +type Checker struct { + Store Store + Ask func(key string, body any) (json.RawMessage, error) + Emit func(event string, body any) error + Now func() time.Time + Logf func(format string, args ...any) + Host string + Registry []Probe + + running sync.Mutex + mu sync.Mutex + n uint64 + last *Run + // seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is + // written only when two runs in a row see it, or while it is already written (novox/hq issue 277). + seen map[string]bool + // unsaid counts the runs whose `checked` could not be said since the start. + unsaid int +} + +// LastRun is the last run, nil before the first. +func (c *Checker) LastRun() *Run { + c.mu.Lock() + defer c.mu.Unlock() + return c.last +} + +type result struct { + found []Finding + err error + took time.Duration +} + +// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`. +// One run at a time: a person's run during a scheduled one waits for it. +func (c *Checker) RunOnce(ctx context.Context, why string) Run { + c.running.Lock() + defer c.running.Unlock() + c.mu.Lock() + c.n++ + n := c.n + c.mu.Unlock() + started := c.Now() + run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(), + IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why} + + results := make([]result, len(c.Registry)) + var wg sync.WaitGroup + for i, p := range c.Registry { + if p.Run == nil { + continue + } + wg.Add(1) + go func(i int, p Probe) { + defer wg.Done() + results[i] = c.probe(ctx, p) + }(i, p) + } + wg.Wait() + + written, readErr := c.written() + seen := map[string]bool{} + want := map[string]Finding{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Finding + for i, p := range c.Registry { + v := ProbeVerdict{ID: p.ID} + r := results[i] + switch { + case p.Run == nil: + v.Verdict = VerdictDeferred + v.Error = "deferred: " + p.Deferred + run.Counts.Deferred++ + case r.err != nil: + v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() + run.Counts.FailedToRun++ + blind = append(blind, Finding{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, + Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), + Said: firstLine(r.err.Error())}) + default: + ran[p.ID] = true + for _, f := range r.found { + if f.Kind == "" { + f.Kind = p.Kind + } + f.Source = p.ID + f = worded(f) + f.Observer = Module + key := f.Key() + seen[key] = true + if f.Confirm && !c.seenBefore(key) && !written.has(key) { + v.Unconfirmed = append(v.Unconfirmed, f.Summary) + continue + } + want[key] = f + v.Found = append(v.Found, f.Summary) + } + if len(v.Found) == 0 { + v.Verdict = VerdictPass + run.Counts.Passed++ + } else { + v.Verdict = VerdictFail + run.Counts.Failed++ + } + } + if p.Run != nil { + v.Took = r.took.Round(time.Millisecond).String() + } + run.Probes = append(run.Probes, v) + } + ran[sourceSelf] = true + for _, f := range blind { + f = worded(f) + f.Observer = Module + key := f.Key() + seen[key] = true + if c.seenBefore(key) || written.has(key) { + want[key] = f + } + } + if readErr != nil { + // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. + c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) + } else if err := c.write(written, want, ran); err != nil { + c.Logf("[%s] what this run found is not all written: %v", Module, err) + } + c.mu.Lock() + c.seen = seen + c.mu.Unlock() + + run.At = c.Now().UTC() + run.Took = run.At.Sub(started).Round(time.Millisecond).String() + c.mu.Lock() + run.Unsaid = c.unsaid + c.mu.Unlock() + c.keep(run) + return run +} + +// probe runs one probe within its bound; a panic or an overrun is an error, never a pass. +func (c *Checker) probe(ctx context.Context, p Probe) result { + probing, cancel := context.WithTimeout(ctx, Within) + defer cancel() + began := c.Now() + done := make(chan result, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- result{err: fmt.Errorf("the probe panicked: %v", r)} + } + }() + found, err := p.Run(probing, c) + done <- result{found: found, err: err} + }() + select { + case r := <-done: + r.took = c.Now().Sub(began) + return r + case <-probing.Done(): + return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)} + } +} + +func (c *Checker) seenBefore(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.seen[key] +} + +// writtenFindings are the findings in the bucket, by key, with the source that wrote each and what it says. +type writtenFindings map[string]Finding + +func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } + +// written reads every finding this module has written. +func (c *Checker) written() (writtenFindings, error) { + keys, err := c.Store.Keys(FindingsState) + if err != nil { + return writtenFindings{}, err + } + out := writtenFindings{} + for _, k := range keys { + raw, found, err := c.Store.Get(FindingsState, k) + if err != nil { + return out, err + } + if !found { + continue + } + var f Finding + if err := json.Unmarshal(raw, &f); err != nil { + return out, fmt.Errorf("the finding %s is not readable: %w", k, err) + } + out[k] = f + } + return out, nil +} + +// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding +// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). +func (c *Checker) write(written writtenFindings, want map[string]Finding, ran map[string]bool) error { + var errs []string + keys := make([]string, 0, len(want)) + for k := range want { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + f := want[k] + if was, ok := written[k]; ok && was.Says() == f.Says() { + continue + } + if err := c.Store.Put(FindingsState, k, f); err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + for k, f := range written { + if _, still := want[k]; still || !ran[f.Source] { + continue + } + if err := c.Store.Delete(FindingsState, k); err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} + +// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in +// the log: the watchman's S10 says the self-check silent when `checked` stops. +func (c *Checker) keep(run Run) { + c.mu.Lock() + c.last = &run + c.mu.Unlock() + if err := c.Store.Put(VerdictState, LastKey, run); err != nil { + c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err) + } + if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil { + c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err) + } + if c.Emit == nil { + return + } + if err := c.Emit(CheckedEvent, run); err != nil { + c.mu.Lock() + c.unsaid++ + c.mu.Unlock() + c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v", + Module, run.Run, err) + } +} + +// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314). +const overviewFindings = 20 + +// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all. +func Overview(run Run) Run { + probes := make([]ProbeVerdict, len(run.Probes)) + for i, p := range run.Probes { + total := len(p.Found) + len(p.Unconfirmed) + if total > overviewFindings { + p.FoundCount = len(p.Found) + if len(p.Found) > overviewFindings { + p.Found = p.Found[:overviewFindings:overviewFindings] + } + if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room { + p.Unconfirmed = p.Unconfirmed[:room:room] + } + p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all", + p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID) + } + probes[i] = p + } + run.Probes = probes + return run +} + +// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every. +func (c *Checker) Keep(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-time.After(FirstAfter): + } + tick := time.NewTicker(Every) + defer tick.Stop() + for { + c.RunOnce(ctx, "the schedule") + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go new file mode 100644 index 00000000..42793250 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -0,0 +1,250 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" +) + +// memStore is the bus's buckets in memory, counting the writes. +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + puts map[string]int + keysErr error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) Put(state, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + s.kv[state][key] = raw + s.puts[state+"/"+key]++ + return nil +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.keysErr != nil && state == FindingsState { + return nil, s.keysErr + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) findingKeys() []string { + keys, _ := s.Keys(FindingsState) + return keys +} + +// checker is a self-check over memory, with a registry the test drives. +func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { + var said []string + now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) + c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Emit: func(event string, body any) error { + raw, _ := json.Marshal(body) + said = append(said, event+" "+string(raw)) + return nil + }} + return c, &said +} + +// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that +// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can +// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). +func TestARunWritesFindingsAsTheyChange(t *testing.T) { + var found []Finding + var fail error + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Finding, error) { return found, fail }} + deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} + missing := Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + Summary: "the stream EVENTS is not on the bus"} + behind := Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} + store := newStore() + c, said := checker(store, steady, deferred) + + steps := []struct { + name string + found []Finding + fail error + want []string // finding keys written after the run + verdict string + missingPs int // puts of the missing finding so far + }{ + {"a finding is written", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []Finding{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []Finding{missing, behind}, nil, + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"blind once: clears nothing, held", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, + {"blind twice: its own finding", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1}, + {"no longer seen: cleared", nil, nil, nil, VerdictPass, 1}, + } + for _, s := range steps { + found, fail = s.found, s.fail + run := c.RunOnce(context.Background(), "the test") + if got := store.findingKeys(); !slices.Equal(got, s.want) { + t.Errorf("%s: findings %v, want %v", s.name, got, s.want) + } + if run.Probes[0].Verdict != s.verdict { + t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict) + } + if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") { + t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1]) + } + if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs { + t.Errorf("%s: the unchanged finding was written %d times", s.name, n) + } + } + raw, _, _ := store.Get(FindingsState, "probe.P1.failed") + if raw != nil { + t.Errorf("the blind finding outlived the probe running again: %s", raw) + } + // Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body. + if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) { + t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps)) + } + if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], CheckedEvent+" ") { + t.Fatalf("said %v", *said) + } + var body map[string]any + _ = json.Unmarshal([]byte(strings.TrimPrefix((*said)[0], CheckedEvent+" ")), &body) + for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} { + if _, ok := body[field]; !ok { + t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body) + } + } + last, _, _ := store.Get(VerdictState, LastKey) + var r Run + _ = json.Unmarshal(last, &r) + if r.Counts.Passed != 1 || r.Counts.Deferred != 1 { + t.Errorf("the last verdict counts %+v", r.Counts) + } + // A finding carries the operator's words of its kind, the probe that found it and who observed it. + found, fail = []Finding{missing}, nil + c.RunOnce(context.Background(), "the test") + raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") + var f Finding + _ = json.Unmarshal(raw, &f) + if f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || f.Explanation == "" { + t.Errorf("the written finding is %+v", f) + } + // Blind to its own bucket, a run writes and clears nothing. + store.keysErr = errors.New("the bus is away") + found = nil + c.RunOnce(context.Background(), "the test") + store.keysErr = nil + if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) { + t.Errorf("a run blind to its findings changed them: %v", got) + } +} + +// A probe that panics or overruns its bound could not run: never a pass. +func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { + within := Within + Within = 50 * time.Millisecond + t.Cleanup(func() { Within = within }) + for name, run := range map[string]func(context.Context, *Checker) ([]Finding, error){ + "panic": func(context.Context, *Checker) ([]Finding, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Finding, error) { time.Sleep(time.Second); return nil, nil }, + } { + c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) + c.Now = time.Now + r := c.RunOnce(context.Background(), "the test") + if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 { + t.Errorf("%s: %+v", name, r.Probes[0]) + } + } +} + +// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer +// every state the module keeps (ADR 0564 §12). +func TestTheVerbsAndReads(t *testing.T) { + store := newStore() + many := make([]Finding, 25) + for i := range many { + many[i] = Finding{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + } + c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", + Run: func(context.Context, *Checker) ([]Finding, error) { return many, nil }}) + if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { + t.Fatalf("before a run: %v", err) + } + if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil { + t.Fatal(err) + } + over, _ := c.Doctor(false, "", "") + if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" { + t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More) + } + whole, _ := c.Doctor(false, "d7", "") + if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 { + t.Errorf("one probe whole carries %d findings", len(v.Found)) + } + if _, err := c.Doctor(false, "D99", ""); err == nil { + t.Error("an unknown probe answered") + } + // A fresh checker over the same bucket answers the last verdict from it. + fresh, _ := checker(store) + if _, err := fresh.Doctor(false, "", ""); err != nil { + t.Errorf("the last verdict is not read from its bucket: %v", err) + } + fs, err := c.Findings("") + if err != nil || fs.(map[string]any)["count"] != 25 { + t.Errorf("findings answered %v, %v", fs, err) + } + if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") { + t.Errorf("one finding answered %v, %v", one, err) + } + vs, err := c.Verdicts("", 0) + if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 { + t.Errorf("verdicts answered %v, %v", vs, err) + } + if _, err := c.Verdicts("doctor-1-1", 0); err == nil { + t.Error("an unknown run answered") + } + if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" { + t.Errorf("probes answered %v", p) + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go new file mode 100644 index 00000000..b38fa830 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -0,0 +1,210 @@ +package main + +// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once, +// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the +// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"doctor", "probes"} + +// Tool names: the seat's verbs, and the module's own reads. +const ( + DoctorTool = Seat + ".doctor" + ProbesTool = Seat + ".probes" + FindingsTool = "self_check_findings" + VerdictsTool = "self_check_verdicts" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(c *Checker) []stdio.Tool { + return []stdio.Tool{ + {Name: DoctorTool, + Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " + + "failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " + + "every probe now and answers that run; probe= gives one probe's verdict whole, every finding.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "boolean", "description": "run every probe now"}, + "probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}), + Run: func(a map[string]any) (any, error) { + return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a)) + }}, + {Name: ProbesTool, + Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " + + "it raises, and for one deferred the facts it lacks on the bus.", + Run: func(map[string]any) (any, error) { return c.Probes(), nil }}, + {Name: FindingsTool, + Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " + + "finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}), + Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }}, + {Name: VerdictsTool, + Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " + + "with its counts; or one run whole by its id. The last run is the doctor verb's.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"}, + "limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}), + Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }}, + } +} + +// Doctor is the verb: the last verdict, a run now, or one probe of either. +func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) { + var r Run + switch { + case run: + r = c.RunOnce(context.Background(), why) + case c.LastRun() != nil: + r = *c.LastRun() + default: + raw, found, err := c.Store.Get(VerdictState, LastKey) + if err != nil { + return nil, fmt.Errorf("the last verdict cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+ + "every %s — run=true runs it now", FirstAfter, Every) + } + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("the last verdict is not readable: %w", err) + } + } + if probe != "" { + one, ok := oneProbe(r, probe) + if !ok { + return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool) + } + r = one + } else { + r = Overview(r) + } + return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(), + "note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " + + "longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil +} + +func oneProbe(r Run, id string) (Run, bool) { + for _, p := range r.Probes { + if strings.EqualFold(p.ID, id) { + r.Probes = []ProbeVerdict{p} + return r, true + } + } + return Run{}, false +} + +// Probes is the registry as the verb answers it. +func (c *Checker) Probes() map[string]any { + var out []map[string]any + for _, p := range c.Registry { + row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind} + if len(p.Raises) > 0 { + row["raises"] = p.Raises + } + if p.Run == nil { + row["deferred"] = p.Deferred + } + out = append(out, row) + } + return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()} +} + +// Findings are the findings written now, or one whole. +func (c *Checker) Findings(key string) (any, error) { + if key != "" { + raw, found, err := c.Store.Get(FindingsState, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no finding is written under %s now", key) + } + return json.RawMessage(raw), nil + } + written, err := c.written() + if err != nil { + return nil, fmt.Errorf("the findings cannot be read: %w", err) + } + rows := []map[string]any{} + for k, f := range written { + rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source, + "summary": f.Summary}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) }) + return map[string]any{"findings": rows, "count": len(rows)}, nil +} + +// Verdicts are the week's runs newest first, or one whole. +func (c *Checker) Verdicts(run string, limit int) (any, error) { + if run != "" { + raw, found, err := c.Store.Get(VerdictsState, run) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run) + } + return json.RawMessage(raw), nil + } + keys, err := c.Store.Keys(VerdictsState) + if err != nil { + return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err) + } + sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) }) + total := len(keys) + if limit > 0 && len(keys) > limit { + keys = keys[:limit] + } + rows := []map[string]any{} + for _, k := range keys { + raw, found, err := c.Store.Get(VerdictsState, k) + if err != nil || !found { + continue + } + var r Run + if json.Unmarshal(raw, &r) == nil { + rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts}) + } + } + return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil +} + +// runOrder orders run ids `doctor--` by time, then number. +func runOrder(id string) string { + parts := strings.Split(id, "-") + if len(parts) != 3 { + return id + } + return fmt.Sprintf("%020s-%020s", parts[1], parts[2]) +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func boolArg(a map[string]any, k string) bool { + b, _ := a[k].(bool) + return b +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go new file mode 100644 index 00000000..8b82b408 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -0,0 +1,290 @@ +//go:build integration + +package main + +// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real +// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from +// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus, +// writes them as findings into its bucket on the bus, keeps the verdict, and says checked. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + } + _ = json.Unmarshal(m.Params, &ask) + if ask.Key != StreamsTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + // The bus module's answer, from the server's own streams. + var detail []map[string]any + for name := range p.js.StreamNames(p.ctx).Name() { + detail = append(detail, map[string]any{"name": name}) + } + p.answer(m.ID, map[string]any{"detail": detail}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus. +func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the self-check does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{FindingsState, VerdictState, VerdictsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil { + t.Fatal(err) + } + } + checked, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + CheckedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(DoctorTool, map[string]any{"run": true}) + if err != nil { + t.Fatal(err) + } + var answer struct { + Run Run `json:"run"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + t.Fatal(err) + } + verdicts := map[string]string{} + for _, v := range answer.Run.Probes { + verdicts[v.ID] = v.Verdict + } + if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun || + verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") { + t.Errorf("the run answered %+v", answer.Run) + } + msg, err := checked.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("checked was not said: %v", err) + } + var said Run + if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run { + t.Errorf("checked said %s", msg.Data) + } + findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState) + e, err := findings.Get(ctx, "bus.EVENTS.stream-missing") + if err != nil { + t.Fatalf("the missing stream is not a finding on the bus: %v", err) + } + var f Finding + if err := json.Unmarshal(e.Value(), &f); err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + t.Errorf("the finding is %s", e.Value()) + } + verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) + if _, err := verdict.Get(ctx, LastKey); err != nil { + t.Errorf("the last verdict is not kept: %v", err) + } + if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) { + t.Errorf("the week's verdicts answered %s, %v", raw, err) + } +} diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod new file mode 100644 index 00000000..abaa962a --- /dev/null +++ b/modules/mesh-self-check/go.mod @@ -0,0 +1,22 @@ +module mesh-self-check + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum new file mode 100644 index 00000000..043e05f0 --- /dev/null +++ b/modules/mesh-self-check/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json new file mode 100644 index 00000000..b0e732fe --- /dev/null +++ b/modules/mesh-self-check/module.json @@ -0,0 +1,67 @@ +{ + "module": "mesh-self-check", + "version": "1", + "runs-as": "mesh-self-check", + "claims": [ + { + "name": "mesh-self-check", + "scope": "mesh", + "serves": [ + "doctor", + "probes" + ] + } + ], + "emits": [ + "checked" + ], + "invokes": [ + "nats.nats_streams", + "nats.nats_closed_connections", + "seat:mesh-watchdogs.signals" + ], + "state": [ + "findings", + "verdict", + { + "name": "verdicts", + "ttl-seconds": 604800 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-self-check", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-self-check", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-self-check" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-self-check", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-self-check", + "binary": "mesh-self-check", + "loads": [ + "mesh-self-check" + ] + } + ] + } +} From babe2f194a39d513fb64bc9d157468817f5caf6a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:48:09 +0100 Subject: [PATCH 02/20] Write the self-check's findings and checked in mesh-sdk's health layout, so the condition keeper reads them as it reads every observer's (issue 1952) Pins the group's mesh-sdk head 69fcb6c. A seat verb's run="true" is taken as the seat schema gives it. --- .../{finding.go => observation.go} | 133 +++++++++--------- .../cmd/mesh-self-check/probes.go | 34 ++--- .../cmd/mesh-self-check/probes_test.go | 4 +- .../cmd/mesh-self-check/selfcheck.go | 94 +++++-------- .../cmd/mesh-self-check/selfcheck_test.go | 40 +++--- .../cmd/mesh-self-check/tools.go | 10 +- .../mesh-self-check/whole_integration_test.go | 5 +- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +- 9 files changed, 153 insertions(+), 173 deletions(-) rename modules/mesh-self-check/cmd/mesh-self-check/{finding.go => observation.go} (50%) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/finding.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go similarity index 50% rename from modules/mesh-self-check/cmd/mesh-self-check/finding.go rename to modules/mesh-self-check/cmd/mesh-self-check/observation.go index 9a20352e..71209ce9 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/finding.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/observation.go @@ -1,95 +1,88 @@ package main -// A finding: what an observer writes for the condition keeper (novox/hq ADR 0564 §2). **A copy of the contract -// mesh-sdk holds (issue 1952, S3)**: its layout is the controller's observation (mesh-controller -// internal/conditions/condition.go), its key the controller's key rule, so a finding written here and the condition -// the controller raised for the same fault share one key. When mesh-sdk's go/health carries the finding, this file -// is replaced by it and nothing else changes. +// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an +// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong +// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key +// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. import ( "encoding/json" - "regexp" - "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) -// The two severities a condition has (to-be 45 §2). +// The severities and scopes, as mesh-sdk's health names them. const ( - Urgent = "urgent" - Warning = "warning" + Urgent = health.Urgent + Warning = health.Warning + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh ) -// The scopes a key starts with, as the controller names them. -const ( - ScopeBus = "bus" - ScopeCore = "core" - ScopeProbe = "probe" - ScopeMesh = "mesh" -) +// Subject is what a finding is about. +type Subject = health.Subject -// Subject is what a finding is about: its scope, its id within the scope, the machine it concerns. -type Subject struct { - Scope string `json:"scope"` - ID string `json:"id"` - Machine string `json:"machine,omitempty"` - Also []string `json:"also,omitempty"` +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is the probe that saw it, or the self-check itself for a probe that could not run. + Source string + // Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277). + Confirm bool } -// Finding is one thing an observer sees wrong now, kept under its key in the observer's bucket `findings` and -// deleted when it is no longer seen. Token and Confirm are the observer's own and never written. -type Finding struct { - Kind string `json:"kind"` - Subject Subject `json:"subject"` - Severity string `json:"severity"` - // Summary is one line in the mesh's words; Said the evidence (an address or a name stays here, never in the - // summary, ADR 0234 §6). - Summary string `json:"summary"` - Said string `json:"said,omitempty"` - // Headline, Explanation, Resolved and Needs are what the operator reads (ADR 0253). - Headline string `json:"headline"` - Explanation string `json:"explanation"` - Resolved string `json:"resolved"` - Needs string `json:"needs,omitempty"` - // Source is what found it: a probe's id, or the observer itself for its own blindness. - Source string `json:"source"` - // Observer is the module that wrote it. - Observer string `json:"observer"` - - Token string `json:"-"` - Confirm bool `json:"-"` -} - -// Key is where the finding is kept: `..`, the controller's key for the same condition. -func (f Finding) Key() string { - token := f.Token +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token if token == "" { - token = f.Kind + token = o.Kind } - return Key(f.Subject.Scope, f.Subject.ID, token) + return health.Key(o.Subject.Scope, o.Subject.ID, token) } -// Says is what the finding says, as written: two findings that say the same are one write (ADR 0289 §1). -func (f Finding) Says() string { +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev raw, _ := json.Marshal(f) return string(raw) } -var unsafeKey = regexp.MustCompile(`[^A-Za-z0-9_=/-]`) - -// Key composes a key from its parts, each token made safe for the bus, as the controller's conditions.Key does. -func Key(scope, id, token string) string { - var parts []string - for _, p := range append(append([]string{scope}, strings.Split(id, ".")...), token) { - p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_") - if p == "" { - p = "_" - } - parts = append(parts, p) - } - return strings.Join(parts, ".") -} - // worded gives a finding the operator's words registered for its kind, where it says none of its own. -func worded(f Finding) Finding { +func worded(f Observation) Observation { w, ok := words[f.Kind] if !ok { return f diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 9f4e9882..44be64ce 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -146,8 +146,8 @@ func expectedConsumers() map[string][]buscompose.Consumer { } // probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. -func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) { - var out []Finding +func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) { + var out []Observation byStream := expectedConsumers() streams := make([]string, 0, len(byStream)) for s := range byStream { @@ -175,21 +175,21 @@ func probeConsumers(ctx context.Context, c *Checker) ([]Finding, error) { who := want.Stream + "." + want.Name have, ok := held[want.Name] if !ok { - out = append(out, Finding{Kind: "consumer-lost", Token: "missing", Severity: Warning, + out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), Said: "no consumer " + want.Name + " on " + want.Stream}) continue } if differs := consumerDiffers(want, have); differs != "" { - out = append(out, Finding{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, + out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), Said: differs}) } if want.Stream != "NODES" && have.Pending > behindBound { // One look at a consumer a burst put behind is not one that stays behind: held for the next run, // so healer H4 resets nothing on a single sample (novox/hq issue 277). - out = append(out, Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeBus, ID: who}, Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) @@ -220,7 +220,7 @@ func consumerDiffers(want buscompose.Consumer, have consumerRow) string { } // probeStreams is D7: every stream the mesh defines exists as defined. -func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) { +func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { rows, err := c.streams("") if err != nil { return nil, err @@ -229,17 +229,17 @@ func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) { for _, r := range rows { held[r.Name] = r } - var out []Finding + var out []Observation for _, s := range buscompose.MeshStreams() { have, ok := held[s.Name] if !ok { - out = append(out, Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), Said: "no stream " + s.Name}) continue } if differs := streamDiffers(s, have); differs != "" { - out = append(out, Finding{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), Said: differs}) } @@ -293,7 +293,7 @@ type closedAnswer struct { } // probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. -func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { +func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) if err != nil { return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) @@ -306,7 +306,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { if hours <= 0 { hours = 1 } - var out []Finding + var out []Observation for _, u := range read.Users { if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { continue // refused before it logged in is nobody's reconnect loop @@ -334,7 +334,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { partial = " (at least: the server's record of closed connections does not reach back the whole hour)" } who, machine := busUserWords(u.User) - out = append(out, Finding{Kind: "bus-reconnects", Severity: Warning, + out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), @@ -370,9 +370,9 @@ func busUserWords(user string) (string, string) { // probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the // watchman's pulse on `checked`, is the other half. -func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) { +func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { - return []Finding{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + "late signal is being said", Said: firstLine(err.Error())}}, nil @@ -383,7 +383,7 @@ func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) { // probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it // to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip, // and no answer is no pass. -func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) { +func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { rows, err := c.streams("") if err != nil { return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) @@ -430,12 +430,12 @@ func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) { if len(problems) == 0 { return nil, nil } - return []Finding{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil } // sorted is findings in a stated order, so two runs over one mesh say the same. -func sorted(out []Finding) []Finding { +func sorted(out []Observation) []Observation { sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) return out } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index 7b918757..825416e0 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -88,7 +88,7 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) { return nil, errors.New("nats: no responders available for request") } -func keysOf(found []Finding) []string { +func keysOf(found []Observation) []string { var out []string for _, f := range found { out = append(out, f.Key()) @@ -100,7 +100,7 @@ func keysOf(found []Finding) []string { func TestTheBusProbes(t *testing.T) { cases := []struct { name string - probe func(context.Context, *Checker) ([]Finding, error) + probe func(context.Context, *Checker) ([]Observation, error) bus bus want []string err string diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go index aa670108..8d07725d 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -15,6 +15,8 @@ import ( "strings" "sync" "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) // Module and Seat are this module and the seat it holds once for the mesh. @@ -25,11 +27,11 @@ const ( // The states it keeps, and the event that ends every run. const ( - FindingsState = "findings" + FindingsState = health.FindingsState VerdictState = "verdict" VerdictsState = "verdicts" LastKey = "last" - CheckedEvent = "checked" + CheckedEvent = health.EventChecked ) // The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). @@ -39,12 +41,12 @@ var ( Within = 30 * time.Second ) -// The verdicts a probe can have. +// The verdicts a probe can have, as mesh-sdk's health names them. const ( - VerdictPass = "pass" - VerdictFail = "fail" - VerdictFailedToRun = "failed-to-run" - VerdictDeferred = "deferred" + VerdictPass = health.VerdictPassed + VerdictFail = health.VerdictFailed + VerdictFailedToRun = health.VerdictFailedToRun + VerdictDeferred = health.VerdictDeferred ) // sourceSelf is what writes a probe's failure to run: the self-check itself. @@ -68,43 +70,16 @@ type Probe struct { // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): // answered deferred, never a pass. Deferred string - Run func(ctx context.Context, c *Checker) ([]Finding, error) + Run func(ctx context.Context, c *Checker) ([]Observation, error) } -// ProbeVerdict is one probe's outcome in a run, as the controller's doctor says it. -type ProbeVerdict struct { - ID string `json:"id"` - Verdict string `json:"verdict"` - Found []string `json:"found,omitempty"` - Unconfirmed []string `json:"unconfirmed,omitempty"` - Error string `json:"error,omitempty"` - Took string `json:"took,omitempty"` - FoundCount int `json:"found-count,omitempty"` - More string `json:"more,omitempty"` -} - -// Counts are a run's verdicts, counted. -type Counts struct { - Passed int `json:"passed"` - Failed int `json:"failed"` - FailedToRun int `json:"failed-to-run"` - Deferred int `json:"deferred"` -} - -// Run is one run of the registry, and the body of `checked`: the controller's doctor-heartbeat body unchanged -// (ADR 0564 §6), Controller naming the machine that ran it. -type Run struct { - Run string `json:"run"` - At time.Time `json:"at"` - Started time.Time `json:"started"` - Took string `json:"took"` - IntervalSeconds int `json:"interval-seconds"` - Counts Counts `json:"counts"` - Probes []ProbeVerdict `json:"probes"` - Controller string `json:"controller"` - Why string `json:"why"` - Unsaid int `json:"unsaid,omitempty"` -} +// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's +// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it. +type ( + ProbeVerdict = health.ProbeVerdict + Counts = health.CheckedCounts + Run = health.Checked +) // Checker is the self-check: the registry and what its probes reach. type Checker struct { @@ -135,7 +110,7 @@ func (c *Checker) LastRun() *Run { } type result struct { - found []Finding + found []Observation err error took time.Duration } @@ -169,9 +144,9 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { written, readErr := c.written() seen := map[string]bool{} - want := map[string]Finding{} // every finding this run writes, by key - ran := map[string]bool{} // the sources whose findings this run judged whole - var blind []Finding + want := map[string]Observation{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Observation for i, p := range c.Registry { v := ProbeVerdict{ID: p.ID} r := results[i] @@ -183,7 +158,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { case r.err != nil: v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() run.Counts.FailedToRun++ - blind = append(blind, Finding{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, + blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), Said: firstLine(r.err.Error())}) @@ -195,7 +170,6 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { } f.Source = p.ID f = worded(f) - f.Observer = Module key := f.Key() seen[key] = true if f.Confirm && !c.seenBefore(key) && !written.has(key) { @@ -221,7 +195,6 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { ran[sourceSelf] = true for _, f := range blind { f = worded(f) - f.Observer = Module key := f.Key() seen[key] = true if c.seenBefore(key) || written.has(key) { @@ -231,7 +204,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { if readErr != nil { // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) - } else if err := c.write(written, want, ran); err != nil { + } else if err := c.write(written, want, ran, run.Run); err != nil { c.Logf("[%s] what this run found is not all written: %v", Module, err) } c.mu.Lock() @@ -277,8 +250,8 @@ func (c *Checker) seenBefore(key string) bool { return c.seen[key] } -// writtenFindings are the findings in the bucket, by key, with the source that wrote each and what it says. -type writtenFindings map[string]Finding +// writtenFindings are the findings in the bucket, by key. +type writtenFindings map[string]health.Finding func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } @@ -297,8 +270,8 @@ func (c *Checker) written() (writtenFindings, error) { if !found { continue } - var f Finding - if err := json.Unmarshal(raw, &f); err != nil { + f, err := health.DecodeFinding(raw) + if err != nil { return out, fmt.Errorf("the finding %s is not readable: %w", k, err) } out[k] = f @@ -308,19 +281,24 @@ func (c *Checker) written() (writtenFindings, error) { // write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding // of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). -func (c *Checker) write(written writtenFindings, want map[string]Finding, ran map[string]bool) error { +func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error { var errs []string keys := make([]string, 0, len(want)) for k := range want { keys = append(keys, k) } sort.Strings(keys) + now := c.Now() for _, k := range keys { - f := want[k] - if was, ok := written[k]; ok && was.Says() == f.Says() { + f := want[k].Finding(run, now) + if was, ok := written[k]; ok && says(was) == says(f) { continue } - if err := c.Store.Put(FindingsState, k, f); err != nil { + raw, err := health.EncodeFinding(f) + if err == nil { + err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { errs = append(errs, k+": "+err.Error()) } } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index 42793250..311bf924 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -11,6 +11,8 @@ import ( "sync" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/health" ) // memStore is the bus's buckets in memory, counting the writes. @@ -91,29 +93,29 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { // could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can // be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). func TestARunWritesFindingsAsTheyChange(t *testing.T) { - var found []Finding + var found []Observation var fail error - steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Finding, error) { return found, fail }} + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }} deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} - missing := Finding{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "the stream EVENTS is not on the bus"} - behind := Finding{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} store := newStore() c, said := checker(store, steady, deferred) steps := []struct { name string - found []Finding + found []Observation fail error want []string // finding keys written after the run verdict string missingPs int // puts of the missing finding so far }{ - {"a finding is written", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"the same again is no write", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"one look can be wrong: held", []Finding{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"seen twice: written", []Finding{missing, behind}, nil, + {"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []Observation{missing, behind}, nil, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, {"blind once: clears nothing, held", nil, errors.New("timed out"), []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, @@ -162,12 +164,12 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { t.Errorf("the last verdict counts %+v", r.Counts) } // A finding carries the operator's words of its kind, the probe that found it and who observed it. - found, fail = []Finding{missing}, nil + found, fail = []Observation{missing}, nil c.RunOnce(context.Background(), "the test") raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") - var f Finding - _ = json.Unmarshal(raw, &f) - if f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || f.Explanation == "" { + f, err := health.DecodeFinding(raw) + if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || + f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { t.Errorf("the written finding is %+v", f) } // Blind to its own bucket, a run writes and clears nothing. @@ -185,9 +187,9 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { within := Within Within = 50 * time.Millisecond t.Cleanup(func() { Within = within }) - for name, run := range map[string]func(context.Context, *Checker) ([]Finding, error){ - "panic": func(context.Context, *Checker) ([]Finding, error) { panic("boom") }, - "overrun": func(ctx context.Context, _ *Checker) ([]Finding, error) { time.Sleep(time.Second); return nil, nil }, + for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){ + "panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil }, } { c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) c.Now = time.Now @@ -202,12 +204,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { // every state the module keeps (ADR 0564 §12). func TestTheVerbsAndReads(t *testing.T) { store := newStore() - many := make([]Finding, 25) + many := make([]Observation, 25) for i := range many { - many[i] = Finding{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} } c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", - Run: func(context.Context, *Checker) ([]Finding, error) { return many, nil }}) + Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }}) if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { t.Fatalf("before a run: %v", err) } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go index b38fa830..1c1df753 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/tools.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -197,9 +197,15 @@ func strArg(a map[string]any, k string) string { return strings.TrimSpace(s) } +// boolArg reads a flag as a boolean or as the seat's "true". func boolArg(a map[string]any, k string) bool { - b, _ := a[k].(bool) - return b + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false } func intArg(a map[string]any, k string, otherwise int) int { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go index 8b82b408..cc078779 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -26,6 +26,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/testbus" ) @@ -276,8 +277,8 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { if err != nil { t.Fatalf("the missing stream is not a finding on the bus: %v", err) } - var f Finding - if err := json.Unmarshal(e.Value(), &f); err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + f, err := health.DecodeFinding(e.Value()) + if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { t.Errorf("the finding is %s", e.Value()) } verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index abaa962a..a4d5f850 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 043e05f0..ae67790f 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 4c11c2c0c0fb8e18539d4faef62b1783bf9b2828 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:48:09 +0100 Subject: [PATCH 03/20] Move the healers out of the controller into mesh-healers, acting on the keeper's events and only once the lease names no controller (issue 1952) Until the lease write each act is written to would, kept a week by its bucket's age, so a would-act counts for no budget or brake. Acts begin by a create in attempts; H3 asks the grants part's assert; the brake and S15 are its own findings. --- modules/mesh-healers/.gitignore | 2 + modules/mesh-healers/README.md | 37 ++ .../mesh-healers/cmd/mesh-healers/healers.go | 602 ++++++++++++++++++ .../cmd/mesh-healers/healers_test.go | 377 +++++++++++ modules/mesh-healers/cmd/mesh-healers/main.go | 92 +++ .../cmd/mesh-healers/manifest_test.go | 91 +++ .../cmd/mesh-healers/observation.go | 77 +++ .../cmd/mesh-healers/reads-outside | 2 + .../mesh-healers/cmd/mesh-healers/tools.go | 186 ++++++ .../mesh-healers/cmd/mesh-healers/wanted.go | 152 +++++ .../mesh-healers/whole_integration_test.go | 416 ++++++++++++ modules/mesh-healers/go.mod | 22 + modules/mesh-healers/go.sum | 27 + modules/mesh-healers/module.json | 78 +++ 14 files changed, 2161 insertions(+) create mode 100644 modules/mesh-healers/.gitignore create mode 100644 modules/mesh-healers/README.md create mode 100644 modules/mesh-healers/cmd/mesh-healers/healers.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/healers_test.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/main.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/manifest_test.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/observation.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/reads-outside create mode 100644 modules/mesh-healers/cmd/mesh-healers/tools.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/wanted.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go create mode 100644 modules/mesh-healers/go.mod create mode 100644 modules/mesh-healers/go.sum create mode 100644 modules/mesh-healers/module.json diff --git a/modules/mesh-healers/.gitignore b/modules/mesh-healers/.gitignore new file mode 100644 index 00000000..0bafa572 --- /dev/null +++ b/modules/mesh-healers/.gitignore @@ -0,0 +1,2 @@ +/mesh-healers +/cmd/mesh-healers/mesh-healers diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md new file mode 100644 index 00000000..0431657c --- /dev/null +++ b/modules/mesh-healers/README.md @@ -0,0 +1,37 @@ +# mesh-healers + +Holds the seat `mesh-healers` (novox/hq ADR 0564 §7, §8): the mesh's healers, moved out of the controller. Go, one +binary (`cmd/mesh-healers`), served by a runtime of its own on its own bus account (`runs-as`). + +## What it does + +It hears the condition keeper's `condition-raised` and `condition-changed` (seat `mesh-conditions`) on a durable +consumer. A condition whose kind a healer answers is acted on within the healer's budget, its settle and the mesh-wide +brake (twelve acts an hour). A repair is a call of the seat or tool that owns the act: H3 asks the grants part's +`grants_assert` for the module whose consumer is gone. H4 (the controller's own consumer) and H5 (the provider's own +repair) are registered and run elsewhere. + +Each act is begun by a create of its key in `attempts` before it is made and finished after, so a holder that dies +mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's +`tried`, resolver and escalation from `attempts`; a healer writes no condition. + +**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would +take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. + +S15: it watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a +fortnight; a drill and a person's decision by design never count. + +## What it keeps + +- `mesh-healers_attempts`: every act and escalation, kept a week (the bucket's age). +- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease, + kept a week (the bucket's age); nothing else deletes it, and nothing is written there once the lease is handed away. +- `mesh-healers_findings`: the brake, its own blindness, and S15. + +## Its verbs and tools + +- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why; + `recount=true` counts the hand-act log again. +- `mesh-healers.heal`: one act now on an open condition, by key and why. +- `mesh-healers.drill`: refused until the hand-act log's home is delivered (issue 1953), naming `mesh-controller.drill`. +- `healers_findings`, `healers_would`: the reads of its buckets. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go new file mode 100644 index 00000000..170f0442 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -0,0 +1,602 @@ +package main + +// The healers (novox/hq ADR 0564 §7, §8; ADR 0231): a known failure heals itself, under a brake, and every repair is +// said. The registry is compiled in; mesh-healers hears `condition-raised` and `condition-changed` of the seat +// mesh-conditions on a durable consumer, instead of looking every half minute. A repair is a call of the seat or tool +// that owns the act (H3: the grants part's grants_assert). Each act is begun by a create of its key in the bucket +// `attempts` before it is made and finished after, so a holder that dies mid-act has spent it and a second instance +// cannot begin the same act; budgets, settles and the mesh-wide brake count from there. Success is never the +// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`, +// the resolver and the escalation into the condition from `attempts`. +// +// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is +// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches +// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once +// the lease is handed away nothing is written there again. + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-healers" + Seat = "mesh-healers" +) + +// The states it keeps and reads, and the event it says. +const ( + AttemptsState = health.AttemptsState + WouldState = health.WouldState + FindingsState = health.FindingsState + LeaseRead = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey + ConditionsRead = "mesh-conditions.conditions" + HandActsRead = "mesh-controller.hand-acts" + ActedEvent = health.EventHealerActed + // The events it hears: the keeper's, on the seat mesh-conditions. + RaisedEvent = "mesh-conditions.condition-raised" + ChangedEvent = "mesh-conditions.condition-changed" + // AssertTool is the grants part's tool that asserts a module's buckets, logs and streams again. + AssertTool = "mesh-grants.grants_assert" +) + +// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour. +const ( + BrakeLimit = 12 + BrakeWindow = time.Hour +) + +// The outcomes an attempt has. +const ( + OutcomeActing = health.OutcomeActing + OutcomeActed = health.OutcomeActed + OutcomeFailed = health.OutcomeFailed + OutcomeEscalated = health.OutcomeEscalated +) + +// Store is the bus's buckets as this module reaches them, through its runtime. Create writes a key only when it has +// none, and says ErrExists otherwise. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + Create(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside. +type Condition struct { + Key string `json:"key"` + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity string `json:"severity"` + Resolver string `json:"resolver"` + Tried []json.RawMessage `json:"tried,omitempty"` +} + +// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until +// observation clears it. +func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 } + +// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key +// `..` is the healers' own), an act written to `would` while the lease names a +// controller (one key per healer and condition, `.`, its newest), and the body of healer-acted, +// epoch left out (ADR 0564 Limits 10). +type ( + Attempt = health.Attempt + Would = health.Would + Acted = health.HealerActed +) + +// Healer is one row of the registry. +type Healer struct { + ID string + Kinds []string + Condition string + Repair string + Then string + From string + Budget int + Window time.Duration + Settle time.Duration + // ActsIn is where the repair runs: here, or a module that repairs its own (H5), or the controller (H4). + ActsIn string + // applies says whether this condition is one it may act on, and what its budget counts against; why when not. + applies func(c Condition) (budget string, ok bool, why string) + // repair is the act: what it did, what came of it, or why it could not. + repair func(h *Healers, c Condition) (act, said string, err error) +} + +// actsHere marks a healer whose repair mesh-healers makes. +const actsHere = "mesh-healers" + +// Registry is to-be 45 §7's table, compiled in: every healer the mesh has, the ones mesh-healers runs with their act. +func Registry() []Healer { + return []Healer{ + {ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"}, + Condition: "a durable consumer of a module the bus does not hold (D6), or a seat's holder that does not answer (D3)", + Repair: "assert the module's buckets, logs and streams again through the grants part (grants_assert)", + Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)", + Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsHere, + applies: appliesToAModulesObject, repair: assertAgain}, + {ID: "H4", Kinds: []string{"consumer-behind"}, + Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable", + Repair: "re-make the consumer to deliver from now; the one marked resettable is the controller's own", + Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)", + Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, + ActsIn: "the controller, until its lease: the one consumer marked resettable is its own (ADR 0564 §7)"}, + {ID: "H5", Kinds: []string{"provider-failing"}, + Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing)", + Repair: "the provider repairs it itself through the server's own bootstrap command (ADR 0224 §5)", + Then: "announced failing by the provider, braked from ten minutes doubling to six hours", + From: "the identity provider's admin reset through its bootstrap command (179)", + Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute, + ActsIn: "the provider module (keycloak), ADR 0224 §5"}, + } +} + +// Healers is mesh-healers: the registry, what its acts reach, and what it holds. +type Healers struct { + Store Store + Ask func(key string, body any) (json.RawMessage, error) + Emit func(event string, body any) error + Now func() time.Time + Logf func(format string, args ...any) + Registry []Healer + // After wakes the healers when a settle ends, to judge the condition again; nil wakes nothing. + After func(d time.Duration, f func()) + + mu sync.Mutex + // lease is the controller's lease as last read: acting is on only while it is handed away. + acting bool + leaseWhy string + // declined is why each condition was last passed over, said once. + declined map[string]string + // acts are the hand-act log as watched, for S15. + acts map[string]HandAct + // woken is the settle ends already waited for, by budget key. + woken map[string]time.Time + s15At time.Time +} + +// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564 +// §8, Limits 8, as ADR 0561 §3's reader reads it). +func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) { + on, why := buscompose.LeaseHandedAway(raw, found, readErr) + h.mu.Lock() + flipped := on != h.acting || h.leaseWhy == "" + h.acting, h.leaseWhy = on, leaseWords(on, why) + h.mu.Unlock() + if flipped { + h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy) + } +} + +// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting. +func leaseWords(on bool, why string) string { + if on { + return "the controller's lease is handed away: the healers act" + } + if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") { + return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would" + } + return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would" +} + +// Acting says whether the healers act, and why. +func (h *Healers) Acting() (bool, string) { + h.mu.Lock() + defer h.mu.Unlock() + if h.leaseWhy == "" { + return false, "the controller's lease has not been read yet: every act is written as would" + } + return h.acting, h.leaseWhy +} + +// healerFor is the healer that acts here on a kind. +func (h *Healers) healerFor(kind string) (Healer, bool) { + for _, r := range h.Registry { + if r.repair != nil && slices.Contains(r.Kinds, kind) { + return r, true + } + } + return Healer{}, false +} + +// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over. +func (h *Healers) Heard(body []byte) error { + var c Condition + if err := json.Unmarshal(body, &c); err != nil { + h.Logf("[%s] a condition event is not readable, and is passed over: %v", Module, err) + return nil + } + _, err := h.Judge(c, "") + return err +} + +// Judge is one condition judged by the healer registered for its kind. by names a person's `heal`, empty for an +// event. It answers what was done, in words. +func (h *Healers) Judge(c Condition, by string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + row, ok := h.healerFor(c.Kind) + if !ok { + return "no healer here answers the kind " + c.Kind, nil + } + if c.Escalated() { + return "it is handed to the operator already: no healer touches it until observation clears it", nil + } + budget, applies, why := row.applies(c) + if !applies { + h.decline(c.Key, row.ID, why) + return fmt.Sprintf("healer %s leaves it alone: %s", row.ID, why), nil + } + delete(h.declined, c.Key) + now := h.Now() + if !h.acting { + w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()} + if err := h.Store.Put(WouldState, row.ID+"."+c.Key, w); err != nil { + return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err) + } + return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil + } + all, err := h.attempts() + if err != nil { + h.blind(err) + return "", fmt.Errorf("the healers cannot read what they did, so they count no budget and act on nothing: %w", err) + } + h.see("blind", nil) + acts := actsWithin(all, now.Add(-BrakeWindow)) + if braked, said := h.brakeHolds(acts, now); braked { + h.see("brake", []Observation{brakeFinding(acts, said)}) + return "every healer has stopped: " + said, nil + } + h.see("brake", nil) + spent := spentOn(all, row, budget, now) + if n := len(spent); n > 0 && now.Sub(spent[n-1].Begun) < row.Settle { + h.wake(budget, spent[n-1].Begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s waits: its last act on it is still the observation's to judge until %s", row.ID, + spent[n-1].Begun.Add(row.Settle).UTC().Format(time.RFC3339)), nil + } + if len(spent) >= row.Budget { + return h.escalate(row, c, budget, spent) + } + if len(acts) >= BrakeLimit { + return "the brake holds from the next act on", nil + } + return h.act(row, c, budget, len(spent), by) +} + +func (h *Healers) whyNot() string { + if h.leaseWhy == "" { + return "the controller's lease has not been read yet" + } + return h.leaseWhy +} + +// act is one healer's act: begun by a create in attempts, made, finished, and said. +func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) { + begun := h.Now().UTC() + key := attemptKey(row.ID, c.Key, begun) + if by == "" { + by = "healer " + row.ID + } + window := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: row.Repair, Outcome: OutcomeActing, + Begun: begun} + if by != "healer "+row.ID { + a.Said = "asked by " + by + } + if err := h.Store.Create(AttemptsState, key, a); err != nil { + if errors.Is(err, ErrExists) { + return "another instance began this act first", nil + } + return "", fmt.Errorf("healer %s did not act on %s: its attempt could not be begun: %w", row.ID, c.Key, err) + } + act, said, err := row.repair(h, c) + a.Outcome = OutcomeActed + if err != nil { + a.Outcome, said = OutcomeFailed, err.Error() + } + if act != "" { + a.Act = act + } + finished := h.Now().UTC() + if a.Said != "" { + said = a.Said + "; " + said + } + a.Said, a.Finished = said+" ("+window+")", &finished + if err := h.Store.Put(AttemptsState, key, a); err != nil { + h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window}) + h.wake(budget, begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s %s %s: %s — %s (%s)", row.ID, a.Outcome, c.Key, a.Act, said, window), nil +} + +// escalate is a spent budget: the condition is the operator's now. Written once as an attempt the keeper turns into +// the condition's resolver and urgency (ADR 0564 §7); an escalated condition is passed over until it clears. +func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attempt) (string, error) { + var tried []string + for _, s := range spent { + tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.Begun.UTC().Format("15:04 MST"), firstLine(s.Said))) + } + said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window, + c.Key, strings.Join(tried, "; ")) + now := h.Now().UTC() + window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator", + Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now} + if err := h.Store.Create(AttemptsState, attemptKey(row.ID, c.Key, now), a); err != nil && !errors.Is(err, ErrExists) { + return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated, + Said: said, Budget: window}) + return fmt.Sprintf("healer %s handed %s to the operator: %s", row.ID, c.Key, said), nil +} + +// tell says a heal as healer-acted. Not said is said in the log: the attempt holds it. +func (h *Healers) tell(e Acted) { + e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer + e.Show = "mesh-conditions.conditions key=" + e.Condition + if h.Emit == nil { + return + } + if err := h.Emit(ActedEvent, e); err != nil { + h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err) + } +} + +// wake judges a condition again when its settle ends: the one wait of the healers, set by an act, never a look on a +// schedule. The settle is read from the attempt's time, so a wake lost to a restart is the next event's to make. +func (h *Healers) wake(budget string, at time.Time, key string) { + if h.After == nil || !h.woken[budget].Before(at) { + return + } + if h.woken == nil { + h.woken = map[string]time.Time{} + } + h.woken[budget] = at + h.After(at.Sub(h.Now())+time.Second, func() { h.again(key) }) +} + +// again judges a condition again from the keeper's bucket, if it is still open. +func (h *Healers) again(key string) { + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil || !found { + return + } + if _, err := h.Judge(conditionOf(raw), ""); err != nil { + h.Logf("[%s] %s judged again at its settle's end: %v", Module, key, err) + } +} + +func conditionOf(raw []byte) Condition { + var c Condition + _ = json.Unmarshal(raw, &c) + return c +} + +// decline remembers why a healer passed a condition over, said once. +func (h *Healers) decline(key, healer, why string) { + if h.declined == nil { + h.declined = map[string]string{} + } + if h.declined[key] != why { + h.Logf("[%s] healer %s leaves %s alone: %s", Module, healer, key, why) + } + h.declined[key] = why +} + +func attemptKey(healer, condition string, begun time.Time) string { + return healer + "." + condition + "." + strconv.FormatInt(begun.UnixNano(), 10) +} + +// attempts are every attempt kept, oldest first. +func (h *Healers) attempts() ([]Attempt, error) { + keys, err := h.Store.Keys(AttemptsState) + if err != nil { + return nil, err + } + var out []Attempt + for _, k := range keys { + raw, found, err := h.Store.Get(AttemptsState, k) + if err != nil { + return nil, err + } + if !found { + continue + } + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the attempt %s is not readable: %w", k, err) + } + out = append(out, a) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Begun.Before(out[j].Begun) }) + return out, nil +} + +// actsWithin are the attempts since a moment that were acts: an escalation is a saying, and the brake does not count it. +func actsWithin(all []Attempt, since time.Time) []Attempt { + var out []Attempt + for _, a := range all { + if !a.Begun.Before(since) && a.Outcome != OutcomeEscalated { + out = append(out, a) + } + } + return out +} + +// spentOn is one healer's acts against one budget key within its window, oldest first. +func spentOn(all []Attempt, row Healer, budget string, now time.Time) []Attempt { + var out []Attempt + for _, a := range actsWithin(all, now.Add(-row.Window)) { + if a.Healer == row.ID && a.BudgetKey == budget { + out = append(out, a) + } + } + return out +} + +// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake already written +// and an act within the hour still — it lets go an hour after the last act, not the first. +func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) { + said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), BrakeWindow, BrakeLimit) + if len(acts) >= BrakeLimit { + return true, said + } + _, held, _ := h.Store.Get(FindingsState, brakeKey) + if held && len(acts) > 0 { + last := acts[len(acts)-1].Begun + return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said, + last.Add(BrakeWindow).UTC().Format("15:04 MST")) + } + return false, said +} + +var brakeKey = health.Key(ScopeMesh, "healers", "braked") + +// brakeFinding is the brake, as the urgent finding that says it. +func brakeFinding(acts []Attempt, said string) Observation { + counts := map[string]int{} + for _, a := range acts { + counts[a.Healer+" on "+a.Condition]++ + } + var most []string + for what, n := range counts { + most = append(most, fmt.Sprintf("%s ×%d", what, n)) + } + sort.Strings(most) + return Observation{Kind: "healers-braked", Token: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"}, + Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " + + "healer looping, not repairing — " + said, + Said: strings.Join(most, ", "), + Headline: "All automatic repairs stopped", + Needs: "read the details and decide whether the repairs may run again.", + Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " + + "like a loop, so all of them stopped.", + Resolved: "Automatic repairs run again"} +} + +// blind writes the healers' own blindness: they cannot read what they did. +func (h *Healers) blind(err error) { + h.see("blind", []Observation{{Kind: "probe-failed", Token: "failed", Severity: Warning, + Subject: Subject{Scope: ScopeProbe, ID: "healers"}, + Summary: "the healers cannot read what they did, so they count no budget and act on nothing", + Said: firstLine(err.Error()), Headline: "A check of the mesh could not run", + Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.", + Resolved: "The automatic repairs read what they did again"}}) +} + +// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it +// no longer sees (ADR 0564 §2). +func (h *Healers) see(source string, found []Observation) { + now := h.Now() + want := map[string]health.Finding{} + for _, o := range found { + o.Source = source + want[o.Key()] = o.Finding("", now) + } + keys, err := h.Store.Keys(FindingsState) + if err != nil { + h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err) + return + } + for _, k := range keys { + raw, found, err := h.Store.Get(FindingsState, k) + if err != nil || !found { + continue + } + was, err := health.DecodeFinding(raw) + if err != nil || was.Source != source { + continue + } + if f, still := want[k]; still { + if says(was) == says(f) { + delete(want, k) + } + continue + } + if err := h.Store.Delete(FindingsState, k); err != nil { + h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err) + } + } + for k, f := range want { + raw, err := health.EncodeFinding(f) + if err == nil { + err = h.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + h.Logf("[%s] the finding %s is not written: %v", Module, k, err) + } + } +} + +// --- H3 -------------------------------------------------------------------------------------------------------- + +// moduleOf is the module whose objects a condition names: a consumer of a module's own (`.`). The +// controller's own consumers are not a module's: the controller asserts them itself until its lease. +func moduleOf(c Condition) (string, string) { + stream, consumer, ok := strings.Cut(c.Subject.ID, ".") + if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" { + return "", "it names no consumer of a module whose objects the grants part asserts" + } + if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") { + return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease" + } + return consumer, "" +} + +// appliesToAModulesObject is H3's: a module's consumer the bus does not hold, its budget per condition. +func appliesToAModulesObject(c Condition) (string, bool, string) { + if _, why := moduleOf(c); why != "" { + return "", false, why + } + return c.Key, true, "" +} + +// assertAgain is H3: the grants part asserts the module's buckets, logs and streams again. +func assertAgain(h *Healers, c Condition) (string, string, error) { + module, why := moduleOf(c) + if module == "" { + return "", "", errors.New(why) + } + raw, err := h.Ask(AssertTool, map[string]any{"module": module}) + if err != nil { + return "", "", fmt.Errorf("the grants part did not assert %s: %w", module, err) + } + var a struct { + Says string `json:"says"` + } + _ = json.Unmarshal(raw, &a) + return "asserted " + module + "'s buckets, logs and streams again through the grants part", + orSaid(a.Says, "the grants part answered") + "; the probe that raised it says on its next run whether it is there", nil +} + +func orSaid(s, otherwise string) string { + if strings.TrimSpace(s) == "" { + return otherwise + } + return s +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go new file mode 100644 index 00000000..b3c4fb6a --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -0,0 +1,377 @@ +package main + +import ( + "encoding/json" + "errors" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + keysErr map[string]error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) keys(state string) []string { + k, _ := s.Keys(state) + return k +} + +func (s *memStore) attempt(t *testing.T, key string) Attempt { + t.Helper() + raw, _, _ := s.Get(AttemptsState, key) + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatalf("%s: %s", key, raw) + } + return a +} + +// world is mesh-healers over memory, with its clock, its asks, what it said and the wakes it set. +type world struct { + h *Healers + store *memStore + now time.Time + asked []string + said []Acted + wakes []time.Duration +} + +func newWorld() *world { + w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)} + w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, + Now: func() time.Time { return w.now }, + Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + w.asked = append(w.asked, key+" "+string(raw)) + return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil + }, + Emit: func(event string, body any) error { + if a, ok := body.(Acted); ok && event == ActedEvent { + w.said = append(w.said, a) + } + return nil + }, + After: func(d time.Duration, _ func()) { w.wakes = append(w.wakes, d) }} + return w +} + +func (w *world) handAway() { + w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) +} + +// lost is a module's consumer the bus does not hold, as D6 raises it. +func lost(module string) Condition { + return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self", + Subject: Subject{Scope: "bus", ID: "EVENTS." + module}} +} + +func body(c Condition) []byte { + raw, _ := json.Marshal(map[string]any{"key": c.Key, "kind": c.Kind, "subject": c.Subject, "severity": c.Severity, + "resolver": c.Resolver, "tried": c.Tried, "event": "condition-raised", "show": "mesh-conditions.conditions key=" + c.Key}) + return raw +} + +// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no +// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8). +func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) { + cases := []struct { + name string + lease func(h *Healers) + acts bool + }{ + {"never read", func(*Healers) {}, false}, + {"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false}, + {"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false}, + {"a controller holds it", func(h *Healers) { + h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil) + }, false}, + {"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true}, + } + for _, tc := range cases { + w := newWorld() + tc.lease(w.h) + if err := w.h.Heard(body(lost("records"))); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState) + if tc.acts { + if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) || + len(w.said) != 1 || w.said[0].Outcome != OutcomeActed { + t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said) + continue + } + a := w.store.attempt(t, attempts[0]) + if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" { + t.Errorf("%s: the attempt is %+v", tc.name, a) + } + continue + } + if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) { + t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said) + } + if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked { + t.Errorf("%s: a would-act moved the brake", tc.name) + } + } +} + +// An act waits out its settle, then its spent budget hands the condition to the operator once; an escalated condition +// is passed over; a second instance cannot begin an act already begun; the brake stops every healer past twelve acts +// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7). +func TestBudgetSettleEscalationAndBrake(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + steps := []struct { + name string + after time.Duration + cond Condition + attempts int + outcome string // the newest said outcome, empty for none new + }{ + {"acts", 0, c, 1, OutcomeActed}, + {"within its settle: waits", 2 * time.Minute, c, 1, ""}, + {"after its settle, budget spent: escalates", 5 * time.Minute, c, 2, OutcomeEscalated}, + {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, ""}, + } + for _, s := range steps { + w.now = w.now.Add(s.after) + before := len(w.said) + if _, err := w.h.Judge(s.cond, ""); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + if n := len(w.store.keys(AttemptsState)); n != s.attempts { + t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts) + } + switch { + case s.outcome == "" && len(w.said) != before: + t.Errorf("%s: said %+v", s.name, w.said[before:]) + case s.outcome != "" && (len(w.said) != before+1 || w.said[before].Outcome != s.outcome): + t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) + } + } + if len(w.wakes) == 0 || w.wakes[0] != 6*time.Minute+time.Second { + t.Errorf("the settle's end wakes %v, want once after 6m1s", w.wakes) + } + // A second instance cannot begin the act an attempt's key already holds. + x := newWorld() + x.handAway() + _ = x.store.Create(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}) + if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 { + t.Errorf("a second instance acted: %q, asked %v", said, x.asked) + } + // The brake: twelve acts in the hour stop every healer, said as an urgent finding. + b := newWorld() + b.handAway() + for i := 0; i < BrakeLimit; i++ { + b.now = b.now.Add(time.Minute) + _ = b.store.Create(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x", + Outcome: OutcomeActed, Begun: b.now}) + } + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 { + t.Errorf("past the brake: %q, asked %v", said, b.asked) + } + raw, braked, _ := b.store.Get(FindingsState, brakeKey) + f, _ := health.DecodeFinding(raw) + if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module { + t.Errorf("the brake is written as %s", raw) + } + b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") { + t.Errorf("the brake let go before an hour after the last act: %q", said) + } + b.now = b.now.Add(6 * time.Minute) + if _, err := b.h.Judge(lost("mail"), ""); err != nil || len(b.asked) != 1 { + t.Errorf("an hour after the last act the healers do not act again: %v, asked %v", err, b.asked) + } + if _, braked, _ := b.store.Get(FindingsState, brakeKey); braked { + t.Error("the brake's finding outlived the brake") + } + // Blind to what they did, they act on nothing and say so. + z := newWorld() + z.handAway() + z.store.keysErr[AttemptsState] = errors.New("the bus is away") + if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 { + t.Errorf("blind, the healers acted: %v", err) + } + if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) { + t.Errorf("blind, the findings are %v", keys) + } +} + +// H3 answers a module's consumer the bus does not hold, never the controller's own or a condition naming no consumer; +// the registry holds every healer the mesh has, H3 run here, H4 and H5 run elsewhere (ADR 0231 §2, ADR 0564 §7). +func TestTheRegistryAndWhatH3Answers(t *testing.T) { + cases := []struct { + c Condition + holds bool + }{ + {lost("records"), true}, + {lost("controller"), false}, + {lost("controller-merges"), false}, + {Condition{Key: "seat.x.silent", Kind: "holder-silent", Subject: Subject{Scope: "seat", ID: "issue-tracker"}}, false}, + {Condition{Key: "bus.EVENTS.records.behind", Kind: "consumer-behind", Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}, false}, + } + for _, tc := range cases { + w := newWorld() + w.handAway() + said, _ := w.h.Judge(tc.c, "") + if acted := len(w.asked) == 1; acted != tc.holds { + t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said) + } + } + runs := map[string]bool{} + for _, r := range Registry() { + if len(r.Kinds) == 0 || r.Budget < 1 || r.Window <= 0 || r.Settle <= 0 || r.Repair == "" || r.ActsIn == "" { + t.Errorf("%s is not a whole row: %+v", r.ID, r) + } + runs[r.ID] = r.repair != nil + } + if !runs["H3"] || runs["H4"] || runs["H5"] || len(runs) != 3 { + t.Errorf("the healers run here are %v, want H3 alone of H3, H4, H5", runs) + } +} + +// S15: a cause repaired by hand twice within a fortnight is a finding; a decision by design never counts; it clears +// when fewer than two remain, and the healers wake when the oldest leaves the fortnight (ADR 0564 Limits 2). +func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { + w := newWorld() + act := func(key, verb, cause string, ago time.Duration) { + raw, _ := json.Marshal(HandAct{ID: key, At: w.now.Add(-ago), By: "person.jochen", Verb: verb, Why: "x", Cause: cause}) + w.h.HandActHeard(key, "put", raw) + } + act("a1", "broker consumer-reset", "consumer-behind", 10*24*time.Hour) + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Fatalf("one act is a finding: %v", keys) + } + act("a2", "broker consumer-reset", "consumer-behind", 24*time.Hour) + act("a3", "hand-act drill", "drill", time.Hour) + act("a4", "hand-act drill", "drill", 2*time.Hour) + act("a5", "conditions silence", "operator-answer", time.Hour) + act("a6", "conditions silence", "operator-answer", time.Hour) + act("a7", "hand-act record", "relogin", 20*24*time.Hour) + act("a8", "hand-act record", "relogin", time.Hour) + want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"} + if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) { + t.Fatalf("findings %v, want %v", keys, want) + } + raw, _, _ := w.store.Get(FindingsState, want[0]) + if !strings.Contains(string(raw), "healer H4 answers this cause") { + t.Errorf("the finding does not say its healer is not enough: %s", raw) + } + if n := len(w.wakes); n == 0 || w.wakes[n-1] != 4*24*time.Hour+time.Second { + t.Errorf("wakes %v, want the oldest act's leaving the fortnight in 4 days", w.wakes) + } + w.now = w.now.Add(4*24*time.Hour + time.Second) + w.h.Wanted() + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Errorf("past the fortnight the finding stays: %v", keys) + } + w.h.HandActHeard("a2", "delete", nil) + if len(w.h.acts) != 7 { + t.Errorf("a deleted entry is kept: %d", len(w.h.acts)) + } +} + +// The verbs: heal acts on an open condition by key and says why it was asked; drill waits for the log's home; the reads +// answer every bucket. +func TestTheVerbsAndReads(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil { + t.Error("heal took no why") + } + if _, err := w.h.Heal("bus.none", "x", "person.jochen"); err == nil { + t.Error("heal acted on no open condition") + } + if _, err := w.h.Heal(c.Key, "it is gone again", "person.jochen"); err != nil { + t.Fatal(err) + } + keys := w.store.keys(AttemptsState) + if len(keys) != 1 || !strings.Contains(w.store.attempt(t, keys[0]).Said, "asked by person.jochen (it is gone again)") { + t.Errorf("heal's attempt is %v", keys) + } + answer, err := w.h.Answer(7, true) + if err != nil { + t.Fatal(err) + } + a := answer.(map[string]any) + if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 { + t.Errorf("healers answered %v", a) + } + var drill error + for _, tool := range Tools(w.h) { + if tool.Name == DrillTool { + _, drill = tool.Run(map[string]any{}) + } + } + if drill == nil || !strings.Contains(drill.Error(), "mesh-controller.drill") { + t.Errorf("drill answered %v", drill) + } + if _, err := w.h.read(WouldState, ""); err != nil { + t.Error(err) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go new file mode 100644 index 00000000..86ec7539 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -0,0 +1,92 @@ +// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its +// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun, +// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds +// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's +// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and watches the +// controller's hand-act log for S15. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Create(state, key string, value any) error { + _, err := stdio.State(state).Create(key, value) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state, key string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch(key, handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + +func main() { + h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Emit: func(event string, body any) error { return stdio.Emit(event, body) }, + After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }} + // The lease first: until it is read the healers write would-acts only. + go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { + h.LeaseHeard(c.Value, c.Op == "put", nil) + return nil + }) + go watched(HandActsRead, "", func(c stdio.StateChange) error { + h.HandActHeard(c.Key, c.Op, c.Value) + return nil + }) + for _, event := range []string{RaisedEvent, ChangedEvent} { + go func(event string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(event, func(e stdio.Envelope) error { return h.Heard(e.Body) }) + if err == nil { + logf("[%s] hears %s", Module, event) + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, event, err, wait) + } + }(event) + } + if err := stdio.Serve("", Tools(h)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go new file mode 100644 index 00000000..86588a98 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1, +// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's +// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each. +// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + for _, c := range []struct { + name string + got, want []string + }{ + {"emits", m.Emits, []string{ActedEvent}}, + {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent}}, + {"invokes", m.Invokes, []string{AssertTool}}, + {"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}}, + } { + if !slices.Equal(c.got, c.want) { + t.Errorf("%s %v, want %v", c.name, c.got, c.want) + } + } + states := map[string]int{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + TTL int `json:"ttl-seconds"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.TTL + } + week := 7 * 24 * 60 * 60 + if states[AttemptsState] != week || states[WouldState] != week { + t.Errorf("attempts kept %d s and would %d s, want a week each: would has no other deleter", states[AttemptsState], + states[WouldState]) + } + readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool, + "an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool} + tools := map[string]bool{} + for _, tool := range Tools(&Healers{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + if name, _, _ := strings.Cut(tool, " "); !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/observation.go b/modules/mesh-healers/cmd/mesh-healers/observation.go new file mode 100644 index 00000000..853550b5 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/observation.go @@ -0,0 +1,77 @@ +package main + +// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their +// blindness and S15, each written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the +// controller's key for the same condition. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is what saw it: the brake, the healers' blindness, or S15. + Source string +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} diff --git a/modules/mesh-healers/cmd/mesh-healers/reads-outside b/modules/mesh-healers/cmd/mesh-healers/reads-outside new file mode 100644 index 00000000..cdc54f27 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-healers/module.json diff --git a/modules/mesh-healers/cmd/mesh-healers/tools.go b/modules/mesh-healers/cmd/mesh-healers/tools.go new file mode 100644 index 00000000..949f4e2a --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/tools.go @@ -0,0 +1,186 @@ +package main + +// The seat's verbs and the module's own reads (novox/hq ADR 0564 §7, §12): `healers` is the registry, what the healers +// did and would do, the brake and whether they act; `heal` takes one act now on a condition, within budget and brake; +// `drill` waits for the hand-act log's home; and a read of each bucket it keeps, so a recovery is a call. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"healers", "heal", "drill"} + +// Tool names. +const ( + HealersTool = Seat + ".healers" + HealTool = Seat + ".heal" + DrillTool = Seat + ".drill" + FindingsTool = "healers_findings" + WouldTool = "healers_would" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(h *Healers) []stdio.Tool { + return []stdio.Tool{ + {Name: HealersTool, + Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " + + "days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " + + "would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " + + "again for S15 (a cause repaired by hand twice in a fortnight).", + Input: obj(map[string]any{ + "days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"}, + "recount": map[string]any{"type": "boolean", "description": "count the hand-act log again now"}}), + Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }}, + {Name: HealTool, + Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " + + "event would have it — or, while the controller holds its lease, the act it would take written as would. " + + "Says what was done.", + Input: obj(map[string]any{ + "key": map[string]any{"type": "string", "description": "the condition's key"}, + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}), + Run: func(a map[string]any) (any, error) { + return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a)) + }}, + {Name: DrillTool, + Description: "A drill: something broken on purpose to see the mesh raise and clear it, recorded in the hand-act " + + "log. Refused until the log's home is delivered (issue 1953): the controller's drill records it meanwhile.", + Input: obj(map[string]any{"why": map[string]any{"type": "string"}}), + Run: func(map[string]any) (any, error) { + return nil, errors.New("a drill is recorded in the hand-act log, whose one writer is the controller until its " + + "home is delivered (issue 1953): use mesh-controller.drill, which records it and S15 never counts") + }}, + {Name: FindingsTool, + Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " + + "each cause repaired by hand twice in a fortnight (S15); or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }}, + {Name: WouldTool, + Description: "What the healers would have done while the controller holds its lease (their bucket would, each " + + "key kept a week): one per healer and condition, its newest; or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }}, + } +} + +// Heal is the verb heal: one condition judged now by its healer. +func (h *Healers) Heal(key, why, caller string) (any, error) { + if key == "" || why == "" { + return nil, errors.New("heal names the condition's key and says why") + } + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil { + return nil, fmt.Errorf("the open conditions cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("no condition is open under %s", key) + } + said, err := h.Judge(conditionOf(raw), caller+" ("+why+")") + if err != nil { + return nil, err + } + return map[string]any{"said": said}, nil +} + +// Answer is the verb healers. +func (h *Healers) Answer(days int, recount bool) (any, error) { + if recount { + h.Wanted() + } + now := h.Now() + all, err := h.attempts() + if err != nil { + return nil, fmt.Errorf("what the healers did cannot be read: %w", err) + } + var recent []Attempt + for _, a := range all { + if now.Sub(a.Begun) <= time.Duration(days)*24*time.Hour { + recent = append(recent, a) + } + } + sort.SliceStable(recent, func(i, j int) bool { return recent[i].Begun.After(recent[j].Begun) }) + braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now) + var rows []map[string]any + for _, r := range h.Registry { + rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair, + "budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle), + "then": r.Then, "acts-in": r.ActsIn, "from": r.From}) + } + acting, why := h.Acting() + would, err := h.read(WouldState, "") + if err != nil { + would = map[string]any{"unread": err.Error()} + } + h.mu.Lock() + declined := map[string]string{} + for k, v := range h.declined { + declined[k] = v + } + h.mu.Unlock() + if recent == nil { + recent = []Attempt{} + } + return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would, + "brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()}, + "declined": declined, + "note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil +} + +// read is every key of a bucket with its value, or one whole. +func (h *Healers) read(state, key string) (any, error) { + if key != "" { + raw, found, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("nothing is kept under %s in %s", key, state) + } + return json.RawMessage(raw), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := h.Store.Get(state, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{state: rows, "count": len(rows)}, nil +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go new file mode 100644 index 00000000..d07ec541 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -0,0 +1,152 @@ +package main + +// S15: a cause repaired by hand twice within a fortnight is a healer wanted (ADR 0231 §27, §28; ADR 0564 Limits 2). +// The hand-act log is the controller's bucket mesh-controller_hand-acts, a key-value bucket with no events: mesh-healers +// watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the +// finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design +// (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the +// fortnight: the healers wake once, when the oldest counted act leaves it. From the log's home (issue 1953) they +// watch that instead. + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" +) + +// Within is how far back a repeated cause counts. +const Within = 14 * 24 * time.Hour + +// HandAct is what S15 reads of one entry of the controller's hand-act log (mesh-controller internal/link HandAct). +type HandAct struct { + ID string `json:"id"` + At time.Time `json:"at"` + By string `json:"by"` + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` + Why string `json:"why"` + Cause string `json:"cause"` + Kind string `json:"kind,omitempty"` +} + +// decided are the verbs whose entries are a person's decision by design, with the causes they hold for (every cause +// when none): the controller's handActVerbs, read as S15 reads them. A verb not listed counts as a repair. +var decided = map[string][]string{ + "warrant": nil, + "seat": {"seat-passed"}, + "conditions silence": {"operator-answer"}, + "hand-act record": {"drill"}, + "secret issued": nil, + "hand-act drill": nil, + "retire approve": nil, + "retire reject": nil, + "cleanup delete": nil, + "dead-letters deliver": nil, + "dead-letters drop": nil, + "collect": nil, + "mirrors": nil, + "move-placement": nil, + "move-config": nil, + "bus upgrade": nil, + "upgrade release-backlog": nil, + "secret accept": nil, + "secret rotate": {"leaked-in-logs"}, + "rotate": {"leaked-in-logs"}, +} + +// personsDecision is whether an act is a person's decision by design, by the verb that recorded it. +func personsDecision(a HandAct) bool { + causes, ok := decided[a.Verb] + if !ok { + return false + } + return len(causes) == 0 || slices.Contains(causes, a.Cause) +} + +// HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again. +func (h *Healers) HandActHeard(key, op string, value []byte) { + h.mu.Lock() + if h.acts == nil { + h.acts = map[string]HandAct{} + } + if op != "put" { + delete(h.acts, key) + } else { + var a HandAct + if json.Unmarshal(value, &a) == nil { + h.acts[key] = a + } + } + h.mu.Unlock() + h.Wanted() +} + +// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, +// deleting the rest; it wakes again when the oldest counted act of a wanted cause leaves the fortnight. +func (h *Healers) Wanted() []Observation { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + byCause := map[string][]HandAct{} + for _, a := range h.acts { + if personsDecision(a) || now.Sub(a.At) > Within { + continue + } + byCause[a.Cause] = append(byCause[a.Cause], a) + } + var causes []string + for c, acts := range byCause { + if len(acts) >= 2 { + causes = append(causes, c) + } + } + sort.Strings(causes) + var found []Observation + var next time.Time + for _, cause := range causes { + acts := byCause[cause] + sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) + var lines []string + for _, a := range acts { + lines = append(lines, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"), + strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why)) + } + newest := acts[len(acts)-1] + wanted := "a healer is wanted for it" + for _, r := range h.Registry { + if slices.Contains(r.Kinds, cause) { + wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its budget "+ + "is not enough", r.ID) + } + } + words := strings.NewReplacer(".", " ", "_", " ").Replace(cause) + found = append(found, Observation{Kind: "healer-wanted", Token: "healer-wanted", Severity: Warning, + Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause}, + Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts), + int(Within.Hours()/24), newest.By, wanted), + Said: strings.Join(lines, "; "), + Headline: capital(words) + " keeps being fixed by hand", + Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+ + "for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)), + Resolved: "Resolved: no more hand repairs of " + words}) + if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) { + next = leaves + } + } + h.see("S15", found) + if h.After != nil && !next.IsZero() && !next.Equal(h.s15At) { + h.s15At = next + h.After(next.Sub(now)+time.Second, func() { h.Wanted() }) + } + return found +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go new file mode 100644 index 00000000..5c7ec972 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -0,0 +1,416 @@ +//go:build integration + +package main + +// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's +// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An +// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the +// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the +// hour the brake stops every healer, said as an urgent finding on the bus. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + asked []string +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +// deliver hands the bundle one event as the runtime does from its consumer. +func (p *player) deliver(key string, body []byte) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("e-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{ + "key": key, "node": "ace", "body": json.RawMessage(body)}}}) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &ask) + p.mu.Lock() + p.asked = append(p.asked, ask.Key+" "+string(ask.Body)) + p.mu.Unlock() + if ask.Key != AssertTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + p.answer(m.ID, map[string]any{"says": "1 the bus lacks: created at this ask"}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +func raised(key, module string) []byte { + raw, _ := json.Marshal(map[string]any{"key": key, "kind": "consumer-lost", "severity": "warning", "resolver": "self", + "subject": map[string]any{"scope": "bus", "id": "EVENTS." + module}, "event": "condition-raised", + "show": health.ShowKeeper(key)}) + return raw +} + +// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes. +func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-healers does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, + "mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + bucket := func(name string) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + t.Fatal(err) + } + return kv + } + keys := func(name string) []string { + k, _ := bucket(name).Keys(ctx) + return k + } + lease := bucket("mesh-controller_lease") + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil { + t.Fatal(err) + } + acted, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + ActedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + healers := func() map[string]any { + raw, err := p.call(HealersTool, map[string]any{}) + if err != nil { + return nil + } + var a map[string]any + _ = json.Unmarshal(raw, &a) + return a + } + eventually(t, "the lease read as a controller's", func() bool { + a := healers() + return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller") + }) + + // The controller holds the lease: the induced failure is written as would, never acted on. + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 }) + if k := keys(Module + "_" + AttemptsState); len(k) != 0 { + t.Fatalf("acted under the controller's lease: %v", k) + } + // The write that hands the lease away: the same failure is acted on. + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "the healers acting", func() bool { return healers()["acting"] == true }) + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the act begun and finished on the bus", func() bool { + k := keys(Module + "_" + AttemptsState) + if len(k) != 1 { + return false + } + e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0]) + return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`) + }) + msg, err := acted.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) { + t.Fatalf("healer-acted was not said: %v", err) + } + p.mu.Lock() + asked := strings.Join(p.asked, "\n") + p.mu.Unlock() + if !strings.Contains(asked, AssertTool+` {"module":"records"}`) { + t.Errorf("the grants part was asked %s", asked) + } + // Eleven more acts within the hour: the twelfth was the limit, and the next condition meets the brake. + attempts := bucket(Module + "_" + AttemptsState) + for i := 0; i < BrakeLimit-1; i++ { + begun := time.Now().Add(-time.Duration(i+1) * time.Minute).UTC() + raw, _ := json.Marshal(Attempt{Healer: "H3", Condition: fmt.Sprintf("bus.EVENTS.m%d.missing", i), BudgetKey: "x", + Outcome: OutcomeActed, Begun: begun}) + if _, err := attempts.Put(ctx, attemptKey("H3", fmt.Sprintf("bus.EVENTS.m%d.missing", i), begun), raw); err != nil { + t.Fatal(err) + } + } + p.deliver(RaisedEvent, raised("bus.EVENTS.mail.missing", "mail")) + eventually(t, "the brake written as a finding on the bus", func() bool { + e, err := bucket(Module+"_"+FindingsState).Get(ctx, brakeKey) + if err != nil { + return false + } + f, err := health.DecodeFinding(e.Value()) + return err == nil && f.Severity == health.Urgent && f.Kind == "healers-braked" + }) + if n := len(keys(Module + "_" + AttemptsState)); n != BrakeLimit { + t.Errorf("past the brake the attempts are %d, want %d", n, BrakeLimit) + } +} diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod new file mode 100644 index 00000000..587a34e4 --- /dev/null +++ b/modules/mesh-healers/go.mod @@ -0,0 +1,22 @@ +module mesh-healers + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum new file mode 100644 index 00000000..ae67790f --- /dev/null +++ b/modules/mesh-healers/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json new file mode 100644 index 00000000..186965ec --- /dev/null +++ b/modules/mesh-healers/module.json @@ -0,0 +1,78 @@ +{ + "module": "mesh-healers", + "version": "1", + "runs-as": "mesh-healers", + "claims": [ + { + "name": "mesh-healers", + "scope": "mesh", + "serves": [ + "healers", + "heal", + "drill" + ] + } + ], + "emits": [ + "healer-acted" + ], + "consumes": [ + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed" + ], + "invokes": [ + "mesh-grants.grants_assert" + ], + "reads": [ + "mesh-controller.lease", + "mesh-conditions.conditions", + "mesh-controller.hand-acts" + ], + "state": [ + { + "name": "attempts", + "ttl-seconds": 604800 + }, + { + "name": "would", + "ttl-seconds": 604800 + }, + "findings" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-healers", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-healers", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-healers" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-healers", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-healers", + "binary": "mesh-healers", + "loads": [ + "mesh-healers" + ] + } + ] + } +} From 885b7a3a8e6ebd5c4c166054d0f5161c25a3e68d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:49:46 +0100 Subject: [PATCH 04/20] Pin checks/health to the group's mesh-sdk 69fcb6c, so it knows the health seats the new modules claim (issue 1952) --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index a4d884a2..30f01e1d 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba diff --git a/checks/health/go.sum b/checks/health/go.sum index fc336ec2..aacdf9f6 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= From 9dd27d2edb9cb16b090cdea95bbc42fbb639f855 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:49:46 +0100 Subject: [PATCH 05/20] Re-arm every running settle from the attempts kept when the healers start, so a restart loses no wake (issue 1952) A settle that ended while they were down is judged at once. The README says the week-long bucket age is the deleter of would and attempts for this release. --- modules/mesh-healers/README.md | 6 +++ .../mesh-healers/cmd/mesh-healers/healers.go | 43 ++++++++++++++++++- .../cmd/mesh-healers/healers_test.go | 39 +++++++++++++---- modules/mesh-healers/cmd/mesh-healers/main.go | 6 ++- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md index 0431657c..b266558b 100644 --- a/modules/mesh-healers/README.md +++ b/modules/mesh-healers/README.md @@ -18,6 +18,12 @@ mid-act has spent it and a second instance cannot begin it again. The condition **Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. +Both buckets age as a whole, each key a week after its last write: the runtime gives a bundle no time to live per +key, so the bucket's age is the deleter for this release. The end of a settle and the end of S15's fortnight wake the +healers through an in-process timer, armed by the act; at start, once the lease is first read, every settle still +running is re-armed from `attempts` and one that ended while they were down is judged at once, and the hand-act log's +watch hands over every entry again, which re-arms S15. So a restart loses no wake. + S15: it watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a fortnight; a drill and a person's decision by design never count. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index 170f0442..e18d8cae 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -362,8 +362,49 @@ func (h *Healers) tell(e Acted) { } } +// Rearm is the healers' start, once the lease is first read: every settle still running is re-armed from the attempts +// kept, and one whose end passed while the healers were down is judged at once, so a restart loses no wake. S15's +// fortnight end is re-armed by the hand-act log's watch, which hands over every current entry when it starts. +func (h *Healers) Rearm() { + all, err := h.attempts() + if err != nil { + h.Logf("[%s] what the healers did cannot be read, so no settle is re-armed: %v", Module, err) + return + } + now := h.Now() + newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first + for _, a := range all { + if a.Outcome != OutcomeEscalated { + newest[a.Healer+" "+a.BudgetKey] = a + } + } + var due []string + h.mu.Lock() + for _, a := range newest { + var row Healer + for _, r := range h.Registry { + if r.ID == a.Healer && r.repair != nil { + row = r + } + } + if row.ID == "" { + continue + } + if end := a.Begun.Add(row.Settle); end.After(now) { + h.wake(a.BudgetKey, end, a.Condition) + } else if now.Sub(a.Begun) <= row.Window { + due = append(due, a.Condition) + } + } + h.mu.Unlock() + sort.Strings(due) + for _, key := range due { + h.again(key) + } +} + // wake judges a condition again when its settle ends: the one wait of the healers, set by an act, never a look on a -// schedule. The settle is read from the attempt's time, so a wake lost to a restart is the next event's to make. +// schedule. The settle is read from the attempt's time, and Rearm sets it again after a restart. func (h *Healers) wake(budget string, at time.Time, key string) { if h.After == nil || !h.woken[budget].Before(at) { return diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index b3c4fb6a..00612cb8 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -98,6 +98,13 @@ type world struct { func newWorld() *world { w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)} + w.fresh() + return w +} + +// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten. +func (w *world) fresh() { + w.wakes = nil w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, Now: func() time.Time { return w.now }, Ask: func(key string, body any) (json.RawMessage, error) { @@ -112,7 +119,6 @@ func newWorld() *world { return nil }, After: func(d time.Duration, _ func()) { w.wakes = append(w.wakes, d) }} - return w } func (w *world) handAway() { @@ -182,23 +188,40 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { w := newWorld() w.handAway() c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) steps := []struct { name string after time.Duration cond Condition attempts int outcome string // the newest said outcome, empty for none new + // restart starts the healers again over the same buckets instead of an event, and wake is the settle end + // they must re-arm (none when zero). + restart bool + wake time.Duration }{ - {"acts", 0, c, 1, OutcomeActed}, - {"within its settle: waits", 2 * time.Minute, c, 1, ""}, - {"after its settle, budget spent: escalates", 5 * time.Minute, c, 2, OutcomeEscalated}, + {"acts", 0, c, 1, OutcomeActed, false, 0}, + {"within its settle: waits", 2 * time.Minute, c, 1, "", false, 0}, + {"a restart within its settle re-arms its end", time.Minute, c, 1, "", true, 3*time.Minute + time.Second}, + {"a restart past its settle judges it at once: escalates", 4 * time.Minute, c, 2, OutcomeEscalated, true, 0}, {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", - Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, ""}, + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", false, 0}, } + var first []time.Duration for _, s := range steps { w.now = w.now.Add(s.after) before := len(w.said) - if _, err := w.h.Judge(s.cond, ""); err != nil { + if s.restart { + if len(first) == 0 { + first = w.wakes + } + w.fresh() + w.handAway() + w.h.Rearm() + if s.wake != 0 && !slices.Equal(w.wakes, []time.Duration{s.wake}) { + t.Errorf("%s: wakes %v, want %v", s.name, w.wakes, s.wake) + } + } else if _, err := w.h.Judge(s.cond, ""); err != nil { t.Fatalf("%s: %v", s.name, err) } if n := len(w.store.keys(AttemptsState)); n != s.attempts { @@ -211,8 +234,8 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) } } - if len(w.wakes) == 0 || w.wakes[0] != 6*time.Minute+time.Second { - t.Errorf("the settle's end wakes %v, want once after 6m1s", w.wakes) + if len(first) == 0 || first[0] != 6*time.Minute+time.Second { + t.Errorf("the settle's end wakes %v, want once after 6m1s", first) } // A second instance cannot begin the act an attempt's key already holds. x := newWorld() diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 86ec7539..2207e3c4 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "os" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" @@ -63,9 +64,12 @@ func main() { Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, Emit: func(event string, body any) error { return stdio.Emit(event, body) }, After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }} - // The lease first: until it is read the healers write would-acts only. + // The lease first: until it is read the healers write would-acts only. Once it is first read, every settle is + // re-armed from the attempts kept, so a restart loses no wake. + var started sync.Once go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { h.LeaseHeard(c.Value, c.Op == "put", nil) + started.Do(func() { go h.Rearm() }) return nil }) go watched(HandActsRead, "", func(c stdio.StateChange) error { From e1cdeebcdf66dbcdb0e69bc4d7c8e73c2579c1bd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:54:21 +0100 Subject: [PATCH 06/20] Add mesh-hand-acts, holder of the seat mesh-hand-acts, so the hand-act log has a home the controller's lease does not hold (issue 1953) Acts are kept 400 days under the runtime-proven caller and said as act-recorded; a warrant is read from the router's own record and recorded once per ask; hand-acts answers over both logs. The act layout is a copy of mesh-sdk's until L5-S2 lands. --- modules/mesh-hand-acts/.gitignore | 2 + modules/mesh-hand-acts/README.md | 23 + .../mesh-hand-acts/cmd/mesh-hand-acts/acts.go | 399 ++++++++++++++++++ .../cmd/mesh-hand-acts/acts_test.go | 213 ++++++++++ .../mesh-hand-acts/cmd/mesh-hand-acts/main.go | 51 +++ .../cmd/mesh-hand-acts/manifest_test.go | 65 +++ .../cmd/mesh-hand-acts/reads-outside | 2 + .../cmd/mesh-hand-acts/tools.go | 110 +++++ .../mesh-hand-acts/whole_integration_test.go | 327 ++++++++++++++ modules/mesh-hand-acts/go.mod | 22 + modules/mesh-hand-acts/go.sum | 27 ++ modules/mesh-hand-acts/module.json | 69 +++ 12 files changed, 1310 insertions(+) create mode 100644 modules/mesh-hand-acts/.gitignore create mode 100644 modules/mesh-hand-acts/README.md create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go create mode 100644 modules/mesh-hand-acts/go.mod create mode 100644 modules/mesh-hand-acts/go.sum create mode 100644 modules/mesh-hand-acts/module.json diff --git a/modules/mesh-hand-acts/.gitignore b/modules/mesh-hand-acts/.gitignore new file mode 100644 index 00000000..2a60d0a0 --- /dev/null +++ b/modules/mesh-hand-acts/.gitignore @@ -0,0 +1,2 @@ +/mesh-hand-acts +/cmd/mesh-hand-acts/mesh-hand-acts diff --git a/modules/mesh-hand-acts/README.md b/modules/mesh-hand-acts/README.md new file mode 100644 index 00000000..c7085dfd --- /dev/null +++ b/modules/mesh-hand-acts/README.md @@ -0,0 +1,23 @@ +# mesh-hand-acts + +Holds the seat `mesh-hand-acts` (novox/hq ADR 0565 §10, §11): the hand-act log, moved out of the controller. Go, one +binary (`cmd/mesh-hand-acts`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes the log. + +## What it keeps + +- `mesh-hand-acts_acts`: every act, one key `act--` (`warrant--` for a warrant) in the layout of the + controller's log, kept 400 days by the bucket's age. Each act is said as `act-recorded`. +- `mesh-hand-acts_warrants`: one key `.` per warrant recorded, made once by create, naming its act. + +## Its verbs + +- `mesh-hand-acts.hand-act`: what, why, cause, the condition addressed, and the act's verb (a repair when not given; + `hand-act drill` for a drill, whose cause is always `drill`). Who did it is the caller the runtime proves. +- `mesh-hand-acts.warranted`: asker and ask. Read from the router's own record of that ask (`messenger.asks`), never + from the caller, and recorded once per ask; an ask still open, ended without a choice or another asker's is refused. +- `mesh-hand-acts.hand-acts`: both logs, this one and the controller's `mesh-controller_hand-acts` until its lease + ends, newest first, with the causes repaired by hand twice within a fortnight; `warrants=true` lists the warrants. + +No act is written to both logs: every recorder calls this seat from its delivery, and the controller keeps its own +acts in its own bucket until it loses its lease. diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go new file mode 100644 index 00000000..26969cb1 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go @@ -0,0 +1,399 @@ +package main + +// The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a +// person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat +// mesh-hand-acts, once for the mesh, and is the one writer of its state `acts` (each act a key that the bucket keeps +// 400 days) and `warrants` (one key `.` made once by create). Each act is said as `act-recorded`. +// +// **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the +// router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who +// chose, through which channel, with which proofs and which answer, once per ask. The controller's bucket +// `mesh-controller_hand-acts` stays its own until it loses its lease and is read, never written: `hand-acts` answers +// over both logs, and no act is written to both (ADR 0565 §11). + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-hand-acts" + Seat = "mesh-hand-acts" +) + +// The states it keeps and reads, and the event it says. +const ( + ActsState = "acts" + WarrantsState = "warrants" + // ControllerActs is the controller's log, read until its lease ends. + ControllerActs = "mesh-controller.hand-acts" + // RouterRecords is the router's record of every ask, `.` (ADR 0259): the warrant is read there. + RouterRecords = "messenger.asks" + RecordedEvent = "act-recorded" +) + +// The verbs an act is recorded under, as the controller's log names them: what S15 reads to tell a repair from a +// person's decision by design. +const ( + VerbRecord = "hand-act record" + VerbDrill = "hand-act drill" + VerbWarrant = "warrant" + // CauseDrill is a drill's cause, and the one cause a repair may not give. + CauseDrill = "drill" + // CauseOperatorAnswer is an act the operator chose on a warrant. + CauseOperatorAnswer = "operator-answer" +) + +// RepeatedWithin is how far back a cause counts as repeated (S15). +const RepeatedWithin = 14 * 24 * time.Hour + +// Act is one entry of the log, in the layout of the controller's (mesh-controller internal/link HandAct), so one +// reader reads both logs alike. **A copy of the layout mesh-sdk holds for an act (issue 1953, L5-S2)**, replaced by it +// when it lands. +type Act struct { + ID string `json:"id"` + At time.Time `json:"at"` + By string `json:"by"` + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` + Why string `json:"why"` + Cause string `json:"cause"` + Condition string `json:"condition,omitempty"` + Kind string `json:"kind,omitempty"` + Carried []string `json:"carried,omitempty"` + Via string `json:"via,omitempty"` + Ask string `json:"ask,omitempty"` + Proofs []string `json:"proofs,omitempty"` + RequestedBy string `json:"requested-by,omitempty"` + Outcome string `json:"outcome,omitempty"` + // Log is which log holds it, set when it is read: this seat's, or the controller's. Never written. + Log string `json:"log,omitempty"` +} + +// Store is the bus's buckets as this module reaches them, through its runtime. Create writes a key only when it has +// none, and says ErrExists otherwise. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + Create(state, key string, value any) error + Keys(state string) ([]string, error) +} + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Log is mesh-hand-acts: the log and what records into it. +type Log struct { + Store Store + Emit func(event string, body any) error + Now func() time.Time + Logf func(format string, args ...any) +} + +var seq atomic.Uint64 + +// actKey is an act's key: its time and a sequence, so the keys list in order, as the controller's do. +func actKey(at time.Time) string { + return "act-" + strconv.FormatInt(at.UnixNano(), 10) + "-" + strconv.FormatUint(seq.Add(1), 10) +} + +// Recorded is what a recorder is answered: the act as kept. +type Recorded struct { + Act Act `json:"act"` + Said string `json:"said"` +} + +// HandAct records one act a caller says it did, by hand or on purpose: what, why, its cause and the condition it +// addresses. by is the caller the runtime proved. verb is the act's verb as the log names it: a repair by default, a +// drill with VerbDrill, or a recorder's own (`conditions silence`, `config set`), which S15 reads as it reads the +// controller's. +func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, error) { + what, why, cause, verb = strings.TrimSpace(what), strings.TrimSpace(why), strings.TrimSpace(cause), strings.TrimSpace(verb) + switch { + case !proven(by): + return Recorded{}, fmt.Errorf("who acts is the caller the runtime proves, and it proved none (%q): nothing was recorded", by) + case what == "": + return Recorded{}, errors.New("a hand act says what was done: nothing was recorded") + case why == "": + return Recorded{}, errors.New("a hand act says why: nothing was recorded") + } + if verb == "" { + verb = VerbRecord + } + if verb == VerbDrill { + cause = CauseDrill + } + switch { + case cause == "": + return Recorded{}, errors.New("a hand act says its cause, in a word or a condition's kind: the word a second act " + + "for the same reason uses, which is how a repair done twice is found. Nothing was recorded") + case strings.EqualFold(cause, CauseDrill) && verb != VerbDrill: + return Recorded{}, errors.New("a drill is not recorded as a repair: record it with verb " + VerbDrill + + ", which no healer is wanted for. Nothing was recorded") + } + at := l.Now().UTC() + act := Act{ID: actKey(at), At: at, By: by, Verb: verb, Args: []string{what}, Why: why, Cause: cause, + Condition: strings.TrimSpace(condition)} + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s did %q, because %q (cause: %s)", act.ID, by, what, + why, cause)}, nil +} + +// record writes one act and says it. An act not said is still kept, and the log says so. +func (l *Log) record(act Act) error { + if err := l.Store.Create(ActsState, act.ID, act); err != nil { + return fmt.Errorf("the act could not be recorded: %w", err) + } + if l.Emit != nil { + if err := l.Emit(RecordedEvent, act); err != nil { + l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err) + } + } + return nil +} + +// routerRecord is what warranted reads of the router's record of an ask. +type routerRecord struct { + State string `json:"state"` + Warrant *asks.Warrant `json:"warrant"` + Ask map[string]any `json:"ask"` +} + +var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// Warrant is one ask's warrant as recorded: the key `.` of `warrants`, naming the act it recorded. +type Warrant struct { + Act string `json:"act"` + At time.Time `json:"at"` + By string `json:"recorded-at-the-word-of"` +} + +// Warranted records an asker's act on the operator's warrant for its ask, read from the router's record, once per ask +// however often it is asked (ADR 0274, ADR 0259). The caller gives which ask and nothing else. +func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) { + switch { + case !proven(caller): + return Recorded{}, fmt.Errorf("who asks is the caller the runtime proves, and it proved none (%q): nothing was recorded", caller) + case !askerModule.MatchString(asker): + return Recorded{}, fmt.Errorf("%q is not a module's name", asker) + case !asks.UsableID(ask): + return Recorded{}, fmt.Errorf("%q is not an ask's id", ask) + } + key := asker + "." + ask + if raw, found, err := l.Store.Get(WarrantsState, key); err != nil { + return Recorded{}, err + } else if found { + return l.already(key, raw) + } + raw, found, err := l.Store.Get(RouterRecords, key) + if err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err) + } + var rec routerRecord + if found { + if err := json.Unmarshal(raw, &rec); err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s is not readable: %w", asker, ask, err) + } + } + w := rec.Warrant + switch { + case rec.State == "open" || rec.State == "held": + return Recorded{}, fmt.Errorf("%s's ask %s is still %s: nobody has answered it. Nothing was recorded", asker, ask, rec.State) + case !found || rec.State == "" || w == nil: + return Recorded{}, fmt.Errorf("the router holds no closed record of %s's ask %s. Nothing was recorded", asker, ask) + case w.Asker != asker || w.Ask != ask: + return Recorded{}, fmt.Errorf("the router's record is for %s's ask %s. Nothing was recorded", w.Asker, w.Ask) + case w.Outcome != asks.OutcomeChosen || w.By == nil: + return Recorded{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record. Nothing was "+ + "recorded", asker, ask, w.Outcome) + case w.AskDigest == "": + return Recorded{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest. Nothing was recorded", asker, ask) + } + at := l.Now().UTC() + act := Act{ID: "warrant-" + asker + "-" + ask, At: w.At.UTC(), Verb: VerbWarrant, + Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)}, + Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker), + By: fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity), + Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs, + RequestedBy: asker + ", recorded at the word of " + caller, + Outcome: "chosen; what " + asker + " did with it is in its own record"} + if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) { + raw, _, gerr := l.Store.Get(WarrantsState, key) + if gerr != nil { + return Recorded{}, gerr + } + return l.already(key, raw) + } else if err != nil { + return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err) + } + if err := l.record(act); err != nil && !errors.Is(err, ErrExists) { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil +} + +// already answers a warrant recorded before; an act lost between the warrant and its act is written now. +func (l *Log) already(key string, raw []byte) (Recorded, error) { + var w Warrant + if err := json.Unmarshal(raw, &w); err != nil { + return Recorded{}, fmt.Errorf("the warrant %s is not readable: %w", key, err) + } + actRaw, found, err := l.Store.Get(ActsState, w.Act) + if err != nil { + return Recorded{}, err + } + if !found { + return Recorded{}, fmt.Errorf("the warrant %s names the act %s, which is not kept: asked again, it is read from "+ + "the router again", key, w.Act) + } + var act Act + _ = json.Unmarshal(actRaw, &act) + return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, nil +} + +func via(w asks.Warrant) string { + if w.By == nil { + return w.Channel + } + v := w.Channel + " (" + w.By.Kind + ")" + if w.By.Verified != "" { + v += ", " + w.By.Verified + } + return v +} + +// proven is a bus user the runtime proved: never empty, never the SDK's unproven word. +func proven(by string) bool { + by = strings.TrimSpace(by) + return by != "" && !strings.EqualFold(by, "unproven") && !strings.HasPrefix(by, "(") +} + +// Listing is what hand-acts answers: every act of both logs within the days asked, newest first, narrowed by cause +// and who, and the causes repaired by hand twice or more within a fortnight. +type Listing struct { + Acts []Act `json:"acts"` + Repeated map[string]int `json:"repeated"` + Days int `json:"days"` + // Unread says which log could not be read, so what is missing is said and never read as nothing. + Unread map[string]string `json:"unread,omitempty"` + Kept map[string]int `json:"kept"` +} + +// HandActs reads both logs. +func (l *Log) HandActs(days int, cause, who string) Listing { + now := l.Now() + since := now.Add(-time.Duration(days) * 24 * time.Hour) + out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}} + var all []Act + for _, log := range []string{ActsState, ControllerActs} { + acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin))) + if err != nil { + if out.Unread == nil { + out.Unread = map[string]string{} + } + out.Unread[log] = err.Error() + continue + } + out.Kept[log] = kept + all = append(all, acts...) + } + for _, a := range all { + if now.Sub(a.At) <= RepeatedWithin && !decision(a) { + out.Repeated[a.Cause]++ + } + if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) { + continue + } + out.Acts = append(out.Acts, a) + } + for c, n := range out.Repeated { + if n < 2 { + delete(out.Repeated, c) + } + } + sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) }) + return out +} + +// read is every act of one log since a moment, and how many the log keeps. A key names its time, so only the keys +// within are read whole. +func (l *Log) read(log string, since time.Time) ([]Act, int, error) { + keys, err := l.Store.Keys(log) + if err != nil { + return nil, 0, err + } + var out []Act + for _, k := range keys { + if at, ok := keyTime(k); ok && at.Before(since) { + continue + } + raw, found, err := l.Store.Get(log, k) + if err != nil { + return nil, len(keys), err + } + if !found { + continue + } + var a Act + if json.Unmarshal(raw, &a) != nil || a.At.Before(since) { + continue + } + a.Log = log + if log == ActsState { + a.Log = Seat + } + out = append(out, a) + } + return out, len(keys), nil +} + +// keyTime is the time an act's key names (`act--`), when it names one. +func keyTime(key string) (time.Time, bool) { + parts := strings.Split(key, "-") + if len(parts) != 3 || parts[0] != "act" { + return time.Time{}, false + } + ns, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(0, ns), true +} + +// decided are the verbs whose acts are a person's decision by design, with the causes they hold for (every cause when +// none): the controller's handActVerbs as S15 reads them. A verb not listed is a repair. +var decided = map[string][]string{ + VerbWarrant: nil, "seat": {"seat-passed"}, "conditions silence": {CauseOperatorAnswer}, VerbRecord: {CauseDrill}, + "secret issued": nil, VerbDrill: nil, "retire approve": nil, "retire reject": nil, "cleanup delete": nil, + "dead-letters deliver": nil, "dead-letters drop": nil, "collect": nil, "mirrors": nil, "move-placement": nil, + "move-config": nil, "bus upgrade": nil, "upgrade release-backlog": nil, "secret accept": nil, + "secret rotate": {"leaked-in-logs"}, "rotate": {"leaked-in-logs"}, +} + +func decision(a Act) bool { + causes, ok := decided[a.Verb] + if !ok { + return false + } + if len(causes) == 0 { + return true + } + for _, c := range causes { + if c == a.Cause { + return true + } + } + return false +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go new file mode 100644 index 00000000..9c0ea2d4 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go @@ -0,0 +1,213 @@ +package main + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "sync" + "testing" + "time" +) + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + keysErr map[string]error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func newLog() (*Log, *memStore, *[]Act, *time.Time) { + store := newStore() + var said []Act + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now }, + Emit: func(event string, body any) error { + if a, ok := body.(Act); ok && event == RecordedEvent { + said = append(said, a) + } + return nil + }} + return l, store, &said, &now +} + +// hand-act records what the proven caller says it did, once, and says it; it refuses what cannot be read back, a +// caller the runtime did not prove, and a drill recorded as a repair (ADR 0565 §10; issue 1953 L5-C1). +func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) { + cases := []struct { + name, by, verb, what, why, cause string + wantVerb, wantCause, err string + }{ + {"a repair", "ace.mesh-config", "", "set dns", "it was wrong", "config", VerbRecord, "config", ""}, + {"a recorder's own verb", "novox.mesh-conditions", "conditions silence", "silenced x", "known", "operator-answer", + "conditions silence", "operator-answer", ""}, + {"a drill takes its cause", "novox.mesh-healers", VerbDrill, "stopped records", "does it raise", "", VerbDrill, CauseDrill, ""}, + {"no proven caller", "unproven", "", "x", "y", "z", "", "", "proves"}, + {"by said empty", "", "", "x", "y", "z", "", "", "proves"}, + {"no why", "person.jochen", "", "x", "", "z", "", "", "says why"}, + {"no what", "person.jochen", "", "", "y", "z", "", "", "says what"}, + {"no cause", "person.jochen", "", "x", "y", "", "", "", "cause"}, + {"a drill as a repair", "person.jochen", "", "x", "y", "drill", "", "", "not recorded as a repair"}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing") + keys, _ := store.Keys(ActsState) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || len(keys) != 1 || keys[0] != r.Act.ID || len(*said) != 1 { + t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said)) + } + var kept Act + raw, _, _ := store.Get(ActsState, keys[0]) + _ = json.Unmarshal(raw, &kept) + if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" || + kept.Args[0] != tc.what || kept.Log != "" { + t.Errorf("%s: kept %+v", tc.name, kept) + } + } +} + +// warranted records the router's word once per ask, never the caller's; it refuses an ask still open, one ended +// without a choice, another asker's, and one the router holds no record of (ADR 0565 §10, ADR 0259). +func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) { + chosen := `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"chosen","label":"Approve", + "by":{"who":"operator","kind":"telegram","identity":"42","verified":"P2"},"channel":"telegram","proofs":["P1","P2"], + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}` + cases := []struct { + name, record, asker, ask, err string + }{ + {"chosen", chosen, "claude-code", "a1", ""}, + {"still open", `{"state":"open"}`, "claude-code", "a1", "still open"}, + {"no record", "", "claude-code", "a1", "no closed record"}, + {"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`, + "claude-code", "a1", "no person chose"}, + {"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1", + "record is for mesh-config"}, + {"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest"}, + {"not a module", chosen, "Claude Code", "a1", "not a module"}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + if tc.record != "" { + _ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record)) + } + r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask) + if tc.err != "" { + if keys, _ := store.Keys(ActsState); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || r.Act.Verb != VerbWarrant || r.Act.Cause != CauseOperatorAnswer || + !strings.Contains(r.Act.By, "operator, as telegram identity 42") || r.Act.Via != "telegram (telegram), P2" || + !strings.Contains(r.Act.RequestedBy, "ace.claude-code") || len(r.Act.Proofs) != 2 { + t.Fatalf("%s: %+v, %v", tc.name, r.Act, err) + } + // Asked again, by another instance: nothing more is recorded. + again, err := l.Warranted("g14.claude-code", "claude-code", "a1") + keys, _ := store.Keys(ActsState) + if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 { + t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said)) + } + } +} + +// hand-acts answers over both logs, newest first, narrowed by days, cause and who, and counts each cause repaired by +// hand twice in a fortnight across both, never a person's decision; a log it cannot read is said, never read as empty. +func TestHandActsReadsBothLogs(t *testing.T) { + l, store, _, now := newLog() + put := func(log, verb, cause, by string, ago time.Duration) { + at := now.Add(-ago) + _ = store.Put(log, actKey(at), Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"}) + } + put(ActsState, VerbRecord, "relogin", "person.jochen", time.Hour) + put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour) + put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour) + put(ActsState, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour) + put(ActsState, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour) + put(ActsState, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour) + put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour) + cases := []struct { + name string + days int + cause, who string + want int + }{ + {"a fortnight", 14, "", "", 6}, + {"a day", 1, "", "", 5}, + {"forty days", 40, "", "", 7}, + {"by cause", 14, "relogin", "", 2}, + {"by who", 14, "", "claude-code", 1}, + } + for _, tc := range cases { + got := l.HandActs(tc.days, tc.cause, tc.who) + if len(got.Acts) != tc.want || len(got.Repeated) != 1 || got.Repeated["relogin"] != 2 || got.Unread != nil { + t.Errorf("%s: %d acts, repeated %v, unread %v", tc.name, len(got.Acts), got.Repeated, got.Unread) + } + for i := 1; i < len(got.Acts); i++ { + if got.Acts[i].At.After(got.Acts[i-1].At) { + t.Errorf("%s: not newest first", tc.name) + } + } + } + if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 { + t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept) + } + store.keysErr[ControllerActs] = errors.New("the bus is away") + if got := l.HandActs(14, "", ""); got.Unread[ControllerActs] == "" || len(got.Acts) != 4 { + t.Errorf("an unreadable log: %+v", got.Unread) + } + w, err := l.Warrants() + if err != nil || w.(map[string]any)["count"] != 0 { + t.Errorf("warrants answered %v, %v", w, err) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go new file mode 100644 index 00000000..497875ea --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go @@ -0,0 +1,51 @@ +// mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, each act kept +// 400 days) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for +// `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Create(state, key string, value any) error { + _, err := stdio.State(state).Create(key, value) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func main() { + l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf, + Emit: func(event string, body any) error { return stdio.Emit(event, body) }} + if err := stdio.Serve("", Tools(l)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go new file mode 100644 index 00000000..79bfe248 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565 +// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, and keeps +// acts and warrants 400 days. Every state it keeps and reads has a tool that reads it (ADR 0565 §15). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Runs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + State []struct { + Name string `json:"name"` + TTL int `json:"ttl-seconds"` + } `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.Runs != Module { + t.Errorf("module %q runs as %q", m.Module, m.Runs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v", m.Claims) + } + if !slices.Equal(m.Emits, []string{RecordedEvent}) || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { + t.Errorf("emits %v, reads %v", m.Emits, m.Reads) + } + readBy := map[string]string{ActsState: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, + RouterRecords: WarrantedTool} + tools := map[string]bool{} + for _, tool := range Tools(&Log{}) { + tools[tool.Name] = true + } + for _, s := range m.State { + if s.TTL != 400*24*60*60 { + t.Errorf("%s is kept %d s, want 400 days", s.Name, s.TTL) + } + if !tools[readBy[s.Name]] { + t.Errorf("the state %s has no tool that reads it", s.Name) + } + } + for _, r := range m.Reads { + if !tools[readBy[r]] { + t.Errorf("the read %s has no tool that answers it", r) + } + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside new file mode 100644 index 00000000..60d6785c --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-hand-acts/module.json diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go new file mode 100644 index 00000000..833db625 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go @@ -0,0 +1,110 @@ +package main + +// The seat's verbs (novox/hq ADR 0565 §10, §15): hand-act and warranted record, hand-acts reads both logs and, with +// warrants, every warrant recorded. A recovery is a call: a warrant whose act was lost is written again by asking +// warranted again, from the router's record. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"hand-act", "hand-acts", "warranted"} + +// Tool names. +const ( + HandActTool = Seat + ".hand-act" + HandActsTool = Seat + ".hand-acts" + WarrantedTool = Seat + ".warranted" +) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +// Tools are what the runtime serves. +func Tools(l *Log) []stdio.Tool { + return []stdio.Tool{ + {Name: HandActTool, + Description: "Record one act done by hand (novox/hq ADR 0565 §10): what was done, why, its cause in a word or a " + + "condition's kind (the word a second act for the same reason uses, which is how a repair done twice is " + + "found) and the condition it addresses. Who did it is the caller the runtime proves. verb is the act's " + + "verb as the log names it: a repair when not given, `hand-act drill` for a drill, or a recorder's own " + + "(`conditions silence`, `config set`). Said as act-recorded.", + Input: obj(map[string]any{"what": str("what was done"), "why": str("why, in a sentence"), + "cause": str("the cause, in a word or a condition's kind"), "condition": str("the condition's key, if any"), + "verb": str("the act's verb in the log; a repair when not given")}, "what", "why"), + Run: func(a map[string]any) (any, error) { + return l.HandAct(stdio.CallerOf(a), arg(a, "verb"), arg(a, "what"), arg(a, "why"), arg(a, "cause"), arg(a, "condition")) + }}, + {Name: HandActsTool, + Description: "What was done by hand lately, over both logs (this seat's and the controller's until its lease " + + "ends), newest first: each act with who, why, its cause and which log holds it, and the causes repaired by " + + "hand twice or more within a fortnight (a healer is wanted). Narrowed by cause and who; warrants=true lists " + + "every warrant recorded instead.", + Input: obj(map[string]any{"days": map[string]any{"type": "number", "description": "how many days back (default 14)"}, + "cause": str("only this cause"), "who": str("only acts whose who contains this"), + "warrants": str("\"true\": every warrant recorded, `.` with its act")}), + Run: func(a map[string]any) (any, error) { + if strings.EqualFold(arg(a, "warrants"), "true") || a["warrants"] == true { + return l.Warrants() + } + return l.HandActs(days(a), arg(a, "cause"), arg(a, "who")), nil + }}, + {Name: WarrantedTool, + Description: "Record an asker's act on the operator's warrant (ADR 0274, ADR 0259): read from the router's own " + + "record of that ask — who chose, through which channel, with which proofs and which answer — never from the " + + "caller, once per ask; refused for an ask still open, ended without a choice, or another asker's.", + Input: obj(map[string]any{"asker": str("the module that asked"), "ask": str("its ask's id")}, "asker", "ask"), + Run: func(a map[string]any) (any, error) { + return l.Warranted(stdio.CallerOf(a), arg(a, "asker"), arg(a, "ask")) + }}, + } +} + +// Warrants is every warrant recorded, by key. +func (l *Log) Warrants() (any, error) { + keys, err := l.Store.Keys(WarrantsState) + if err != nil { + return nil, fmt.Errorf("the warrants cannot be read: %w", err) + } + sort.Strings(keys) + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := l.Store.Get(WarrantsState, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{"warrants": rows, "count": len(rows)}, nil +} + +func arg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func days(a map[string]any) int { + switch v := a["days"].(type) { + case float64: + if v > 0 { + return int(v) + } + case string: + var n int + if _, err := fmt.Sscan(v, &n); err == nil && n > 0 { + return n + } + } + return 14 +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go new file mode 100644 index 00000000..62c4600f --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -0,0 +1,327 @@ +//go:build integration + +package main + +// mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered +// from a real nats-server's key-value buckets, mesh/publish onto its subjects). A recorder's hand act is kept in the +// log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1953/L5-C2: an act recorded through the seat is kept on the bus and heard as act-recorded. +func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-hand-acts does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + ActsState, Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + recorded, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + RecordedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(HandActTool, "ace.mesh-config", map[string]any{"what": "set dns on g14", "why": "it was wrong", + "cause": "config", "by": "someone else"}) + if err != nil { + t.Fatal(err) + } + var r Recorded + if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" { + t.Fatalf("hand-act answered %s", raw) + } + acts, _ := js.KeyValue(ctx, Module+"_"+ActsState) + e, err := acts.Get(ctx, r.Act.ID) + if err != nil || !strings.Contains(string(e.Value()), `"by":"ace.mesh-config"`) { + t.Fatalf("the act is not kept on the bus: %v", err) + } + msg, err := recorded.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) { + t.Fatalf("act-recorded was not said: %v", err) + } + // A warrant the router recorded, asked twice: one act. + asks, _ := js.KeyValue(ctx, "messenger_asks") + if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code", + "outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram", + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil { + t.Fatal(err) + } + for _, caller := range []string{"ace.claude-code", "g14.claude-code"} { + if _, err := p.call(WarrantedTool, caller, map[string]any{"asker": "claude-code", "ask": "a1"}); err != nil { + t.Fatal(err) + } + } + if keys, _ := acts.Keys(ctx); len(keys) != 2 { + t.Errorf("the log keeps %v, want the act and one warrant", keys) + } + listed, err := p.call(HandActsTool, "person.jochen", map[string]any{}) + if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) { + t.Errorf("hand-acts answered %s, %v", listed, err) + } +} diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod new file mode 100644 index 00000000..f79a46d2 --- /dev/null +++ b/modules/mesh-hand-acts/go.mod @@ -0,0 +1,22 @@ +module mesh-hand-acts + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum new file mode 100644 index 00000000..ae67790f --- /dev/null +++ b/modules/mesh-hand-acts/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json new file mode 100644 index 00000000..3e562b9a --- /dev/null +++ b/modules/mesh-hand-acts/module.json @@ -0,0 +1,69 @@ +{ + "module": "mesh-hand-acts", + "version": "1", + "runs-as": "mesh-hand-acts", + "claims": [ + { + "name": "mesh-hand-acts", + "scope": "mesh", + "serves": [ + "hand-act", + "hand-acts", + "warranted" + ] + } + ], + "emits": [ + "act-recorded" + ], + "reads": [ + "mesh-controller.hand-acts", + "messenger.asks" + ], + "state": [ + { + "name": "acts", + "ttl-seconds": 34560000 + }, + { + "name": "warrants", + "ttl-seconds": 34560000 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-hand-acts", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-hand-acts", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-hand-acts" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-hand-acts", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-hand-acts", + "binary": "mesh-hand-acts", + "loads": [ + "mesh-hand-acts" + ] + } + ] + } +} From 5b2b70cd6c73eb4cfee195e6cf03e4d6ad0486b8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:56:42 +0100 Subject: [PATCH 07/20] =?UTF-8?q?Keep=20the=20hand-act=20log=20as=20mesh-h?= =?UTF-8?q?and-acts'=20module=20log,=20the=20stream=20ADR=200565=20=C2=A71?= =?UTF-8?q?0=20names,=20so=20every=20act=20is=20held=20in=20its=20order=20?= =?UTF-8?q?(issue=201953)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warrants stay a bucket and no longer age, so a warrant is recorded once; one whose act was lost is written again from the router's record. hand-acts says the log's fill against its cap. --- modules/mesh-hand-acts/README.md | 9 +- .../mesh-hand-acts/cmd/mesh-hand-acts/acts.go | 116 +++++++++++------- .../cmd/mesh-hand-acts/acts_test.go | 109 +++++++++++++--- .../mesh-hand-acts/cmd/mesh-hand-acts/main.go | 47 +++++-- .../cmd/mesh-hand-acts/manifest_test.go | 31 ++--- .../mesh-hand-acts/whole_integration_test.go | 87 +++++++++++-- modules/mesh-hand-acts/module.json | 12 +- 7 files changed, 309 insertions(+), 102 deletions(-) diff --git a/modules/mesh-hand-acts/README.md b/modules/mesh-hand-acts/README.md index c7085dfd..d01b8b49 100644 --- a/modules/mesh-hand-acts/README.md +++ b/modules/mesh-hand-acts/README.md @@ -6,9 +6,12 @@ alone writes the log. ## What it keeps -- `mesh-hand-acts_acts`: every act, one key `act--` (`warrant--` for a warrant) in the layout of the - controller's log, kept 400 days by the bucket's age. Each act is said as `act-recorded`. -- `mesh-hand-acts_warrants`: one key `.` per warrant recorded, made once by create, naming its act. +- the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream): + every act appended in its order under its key `act--` (`warrant--` for a warrant), in the layout + of the controller's log. A log ages nothing and refuses a new act when full, so every act is held 400 days and longer; + `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`. +- `mesh-hand-acts_warrants`: one key `.` per warrant, made once by create before its act is appended, so of + several instances one records it; a warrant whose act was lost is written again from the router's record. ## Its verbs diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go index 26969cb1..01e09440 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go @@ -2,8 +2,10 @@ package main // The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a // person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat -// mesh-hand-acts, once for the mesh, and is the one writer of its state `acts` (each act a key that the bucket keeps -// 400 days) and `warrants` (one key `.` made once by create). Each act is said as `act-recorded`. +// mesh-hand-acts, once for the mesh, and is the one writer of its log `acts` — a stream, as ADR 0565 §10 has it: +// the mesh's module log (ADR 0297), append-only, every act kept in its order under its key, refusing a new act when +// full rather than dropping an old one, so every act is held 400 days and longer — and of its bucket `warrants` (one +// key `.` made once by create). Each act is said as `act-recorded`. // // **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the // router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who @@ -33,7 +35,7 @@ const ( // The states it keeps and reads, and the event it says. const ( - ActsState = "acts" + ActsLog = "acts" WarrantsState = "warrants" // ControllerActs is the controller's log, read until its lease ends. ControllerActs = "mesh-controller.hand-acts" @@ -80,13 +82,25 @@ type Act struct { Log string `json:"log,omitempty"` } -// Store is the bus's buckets as this module reaches them, through its runtime. Create writes a key only when it has -// none, and says ErrExists otherwise. +// Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only +// when it has none, and says ErrExists otherwise. type Store interface { Get(state, key string) ([]byte, bool, error) - Put(state, key string, value any) error Create(state, key string, value any) error Keys(state string) ([]string, error) + // Append adds an entry to the end of a log; Last is a key's newest entry; Entries every entry in order; Fill how + // many entries and bytes the log holds against its cap. + Append(log, key string, value any) error + Last(log, key string) ([]byte, bool, error) + Entries(log string) ([][]byte, error) + Fill(log string) (Fill, error) +} + +// Fill is how full a log is: its entries, its bytes and its cap in bytes. +type Fill struct { + Entries uint64 `json:"entries"` + Bytes uint64 `json:"bytes"` + MaxBytes uint64 `json:"max-bytes"` } // ErrExists is a create of a key another writer made first. @@ -153,7 +167,7 @@ func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, e // record writes one act and says it. An act not said is still kept, and the log says so. func (l *Log) record(act Act) error { - if err := l.Store.Create(ActsState, act.ID, act); err != nil { + if err := l.Store.Append(ActsLog, act.ID, act); err != nil { return fmt.Errorf("the act could not be recorded: %w", err) } if l.Emit != nil { @@ -192,10 +206,14 @@ func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) { return Recorded{}, fmt.Errorf("%q is not an ask's id", ask) } key := asker + "." + ask - if raw, found, err := l.Store.Get(WarrantsState, key); err != nil { + claimed, found, err := l.Store.Get(WarrantsState, key) + if err != nil { return Recorded{}, err - } else if found { - return l.already(key, raw) + } + if found { + if r, ok, err := l.already(key, claimed); err != nil || ok { + return r, err + } } raw, found, err := l.Store.Get(RouterRecords, key) if err != nil { @@ -229,38 +247,34 @@ func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) { Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs, RequestedBy: asker + ", recorded at the word of " + caller, Outcome: "chosen; what " + asker + " did with it is in its own record"} - if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) { - raw, _, gerr := l.Store.Get(WarrantsState, key) - if gerr != nil { - return Recorded{}, gerr + if claimed == nil { + // The claim first: of any number of instances asking at once, the one whose create lands records the act. + if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) { + return Recorded{Said: "another instance records " + act.ID + " now"}, nil + } else if err != nil { + return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err) } - return l.already(key, raw) - } else if err != nil { - return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err) } - if err := l.record(act); err != nil && !errors.Is(err, ErrExists) { + if err := l.record(act); err != nil { return Recorded{}, err } return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil } -// already answers a warrant recorded before; an act lost between the warrant and its act is written now. -func (l *Log) already(key string, raw []byte) (Recorded, error) { +// already answers a warrant recorded before, when its act is in the log; false when the act was lost between the +// claim and the log, so it is read from the router again and written. +func (l *Log) already(key string, raw []byte) (Recorded, bool, error) { var w Warrant if err := json.Unmarshal(raw, &w); err != nil { - return Recorded{}, fmt.Errorf("the warrant %s is not readable: %w", key, err) + return Recorded{}, false, fmt.Errorf("the warrant %s is not readable: %w", key, err) } - actRaw, found, err := l.Store.Get(ActsState, w.Act) - if err != nil { - return Recorded{}, err - } - if !found { - return Recorded{}, fmt.Errorf("the warrant %s names the act %s, which is not kept: asked again, it is read from "+ - "the router again", key, w.Act) + actRaw, found, err := l.Store.Last(ActsLog, w.Act) + if err != nil || !found { + return Recorded{}, false, err } var act Act _ = json.Unmarshal(actRaw, &act) - return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, nil + return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, true, nil } func via(w asks.Warrant) string { @@ -289,6 +303,8 @@ type Listing struct { // Unread says which log could not be read, so what is missing is said and never read as nothing. Unread map[string]string `json:"unread,omitempty"` Kept map[string]int `json:"kept"` + // Fill is how full this seat's log is against its cap: full, it refuses the next act. + Fill *Fill `json:"fill,omitempty"` } // HandActs reads both logs. @@ -297,7 +313,7 @@ func (l *Log) HandActs(days int, cause, who string) Listing { since := now.Add(-time.Duration(days) * 24 * time.Hour) out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}} var all []Act - for _, log := range []string{ActsState, ControllerActs} { + for _, log := range []string{ActsLog, ControllerActs} { acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin))) if err != nil { if out.Unread == nil { @@ -324,17 +340,38 @@ func (l *Log) HandActs(days int, cause, who string) Listing { } } sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) }) + if f, err := l.Store.Fill(ActsLog); err == nil { + out.Fill = &f + } return out } -// read is every act of one log since a moment, and how many the log keeps. A key names its time, so only the keys -// within are read whole. +// read is every act of one log since a moment, and how many it keeps: this seat's log entry by entry, the +// controller's bucket by its keys, each of which names its time, so only those within are read whole. func (l *Log) read(log string, since time.Time) ([]Act, int, error) { + var out []Act + keep := func(raw []byte, from string) { + var a Act + if json.Unmarshal(raw, &a) != nil || a.At.Before(since) { + return + } + a.Log = from + out = append(out, a) + } + if log == ActsLog { + entries, err := l.Store.Entries(ActsLog) + if err != nil { + return nil, 0, err + } + for _, raw := range entries { + keep(raw, Seat) + } + return out, len(entries), nil + } keys, err := l.Store.Keys(log) if err != nil { return nil, 0, err } - var out []Act for _, k := range keys { if at, ok := keyTime(k); ok && at.Before(since) { continue @@ -343,18 +380,9 @@ func (l *Log) read(log string, since time.Time) ([]Act, int, error) { if err != nil { return nil, len(keys), err } - if !found { - continue + if found { + keep(raw, log) } - var a Act - if json.Unmarshal(raw, &a) != nil || a.At.Before(since) { - continue - } - a.Log = log - if log == ActsState { - a.Log = Seat - } - out = append(out, a) } return out, len(keys), nil } diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go index 9c0ea2d4..b2762666 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go @@ -10,14 +10,75 @@ import ( "time" ) +type entry struct { + key string + raw []byte +} + type memStore struct { mu sync.Mutex kv map[string]map[string][]byte + logs map[string][]entry keysErr map[string]error } func newStore() *memStore { - return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}} + return &memStore{kv: map[string]map[string][]byte{}, logs: map[string][]entry{}, keysErr: map[string]error{}} +} + +func (s *memStore) Append(log, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return err + } + s.logs[log] = append(s.logs[log], entry{key, raw}) + return nil +} + +func (s *memStore) Last(log, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i := len(s.logs[log]) - 1; i >= 0; i-- { + if s.logs[log][i].key == key { + return s.logs[log][i].raw, true, nil + } + } + return nil, false, nil +} + +func (s *memStore) Entries(log string) ([][]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return nil, err + } + var out [][]byte + for _, e := range s.logs[log] { + out = append(out, e.raw) + } + return out, nil +} + +func (s *memStore) Fill(log string) (Fill, error) { + s.mu.Lock() + defer s.mu.Unlock() + return Fill{Entries: uint64(len(s.logs[log])), MaxBytes: 1 << 30}, nil +} + +// logKeys are a log's keys in order. +func (s *memStore) logKeys(log string) []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for _, e := range s.logs[log] { + out = append(out, e.key) + } + return out } func (s *memStore) Get(state, key string) ([]byte, bool, error) { @@ -96,7 +157,7 @@ func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) { for _, tc := range cases { l, store, said, _ := newLog() r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing") - keys, _ := store.Keys(ActsState) + keys := store.logKeys(ActsLog) if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 { t.Errorf("%s: %v, kept %v", tc.name, err, keys) @@ -107,7 +168,7 @@ func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) { t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said)) } var kept Act - raw, _, _ := store.Get(ActsState, keys[0]) + raw, _, _ := store.Last(ActsLog, keys[0]) _ = json.Unmarshal(raw, &kept) if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" || kept.Args[0] != tc.what || kept.Log != "" { @@ -124,25 +185,31 @@ func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) { "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}` cases := []struct { name, record, asker, ask, err string + // claimed is a warrant claimed by an instance that stopped before its act reached the log. + claimed bool }{ - {"chosen", chosen, "claude-code", "a1", ""}, - {"still open", `{"state":"open"}`, "claude-code", "a1", "still open"}, - {"no record", "", "claude-code", "a1", "no closed record"}, + {"chosen", chosen, "claude-code", "a1", "", false}, + {"claimed, its act lost: written now", chosen, "claude-code", "a1", "", true}, + {"still open", `{"state":"open"}`, "claude-code", "a1", "still open", false}, + {"no record", "", "claude-code", "a1", "no closed record", false}, {"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`, - "claude-code", "a1", "no person chose"}, + "claude-code", "a1", "no person chose", false}, {"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1", - "record is for mesh-config"}, - {"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest"}, - {"not a module", chosen, "Claude Code", "a1", "not a module"}, + "record is for mesh-config", false}, + {"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest", false}, + {"not a module", chosen, "Claude Code", "a1", "not a module", false}, } for _, tc := range cases { l, store, said, _ := newLog() if tc.record != "" { _ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record)) } + if tc.claimed { + _ = store.Create(WarrantsState, "claude-code.a1", Warrant{Act: "warrant-claude-code-a1"}) + } r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask) if tc.err != "" { - if keys, _ := store.Keys(ActsState); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 { + if keys := store.logKeys(ActsLog); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 { t.Errorf("%s: %v, kept %v", tc.name, err, keys) } continue @@ -154,7 +221,7 @@ func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) { } // Asked again, by another instance: nothing more is recorded. again, err := l.Warranted("g14.claude-code", "claude-code", "a1") - keys, _ := store.Keys(ActsState) + keys := store.logKeys(ActsLog) if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 { t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said)) } @@ -167,14 +234,19 @@ func TestHandActsReadsBothLogs(t *testing.T) { l, store, _, now := newLog() put := func(log, verb, cause, by string, ago time.Duration) { at := now.Add(-ago) - _ = store.Put(log, actKey(at), Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"}) + a := Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"} + if log == ActsLog { + _ = store.Append(log, actKey(at), a) + return + } + _ = store.Put(log, actKey(at), a) } - put(ActsState, VerbRecord, "relogin", "person.jochen", time.Hour) + put(ActsLog, VerbRecord, "relogin", "person.jochen", time.Hour) put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour) put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour) - put(ActsState, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour) - put(ActsState, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour) - put(ActsState, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour) + put(ActsLog, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour) put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour) cases := []struct { name string @@ -199,7 +271,8 @@ func TestHandActsReadsBothLogs(t *testing.T) { } } } - if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 { + if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 || got.Fill == nil || + got.Fill.Entries != 4 { t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept) } store.keysErr[ControllerActs] = errors.New("the bus is away") diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go index 497875ea..0023c53c 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go @@ -1,6 +1,6 @@ // mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of -// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, each act kept -// 400 days) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for +// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, its module +// log: a stream that keeps every act in order) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for // `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr. package main @@ -26,11 +26,6 @@ func (runtimeStore) Get(state, key string) ([]byte, bool, error) { return e.Value, true, nil } -func (runtimeStore) Put(state, key string, value any) error { - _, err := stdio.State(state).Put(key, value) - return err -} - func (runtimeStore) Create(state, key string, value any) error { _, err := stdio.State(state).Create(key, value) if errors.Is(err, stdio.ErrChanged) { @@ -41,6 +36,44 @@ func (runtimeStore) Create(state, key string, value any) error { func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +func (runtimeStore) Append(log, key string, value any) error { + _, err := stdio.Log(log).Append(key, value) + return err +} + +func (runtimeStore) Last(log, key string) ([]byte, bool, error) { + e, err := stdio.Log(log).Last(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Entries(log string) ([][]byte, error) { + var out [][]byte + for from := uint64(0); ; { + page, err := stdio.Log(log).Read(from, "", 1000) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, e.Value) + } + if page.Next == 0 { + return out, nil + } + from = page.Next + } +} + +func (runtimeStore) Fill(log string) (Fill, error) { + i, err := stdio.Log(log).Info() + if err != nil { + return Fill{}, err + } + return Fill{Entries: i.Entries, Bytes: i.Bytes, MaxBytes: i.MaxBytes}, nil +} + func main() { l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf, Emit: func(event string, body any) error { return stdio.Emit(event, body) }} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go index 79bfe248..90669933 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -9,8 +9,9 @@ import ( ) // The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565 -// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, and keeps -// acts and warrants 400 days. Every state it keeps and reads has a tool that reads it (ADR 0565 §15). +// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, keeps its +// acts as its log (a stream, §10) and warrants as a bucket that never ages, so a warrant is recorded once. Every state +// and log it keeps and reads has a tool that reads it (ADR 0565 §15). func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) if err != nil { @@ -24,12 +25,10 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { Scope string `json:"scope"` Serves []string `json:"serves"` } `json:"claims"` - Emits []string `json:"emits"` - Reads []string `json:"reads"` - State []struct { - Name string `json:"name"` - TTL int `json:"ttl-seconds"` - } `json:"state"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + Logs []string `json:"logs"` + State []json.RawMessage `json:"state"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -43,18 +42,22 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if !slices.Equal(m.Emits, []string{RecordedEvent}) || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { t.Errorf("emits %v, reads %v", m.Emits, m.Reads) } - readBy := map[string]string{ActsState: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, + readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, RouterRecords: WarrantedTool} tools := map[string]bool{} for _, tool := range Tools(&Log{}) { tools[tool.Name] = true } - for _, s := range m.State { - if s.TTL != 400*24*60*60 { - t.Errorf("%s is kept %d s, want 400 days", s.Name, s.TTL) + if !slices.Equal(m.Logs, []string{ActsLog}) { + t.Errorf("logs %v, want the acts as a log", m.Logs) + } + for _, raw := range m.State { + var name string + if json.Unmarshal(raw, &name) != nil || name != WarrantsState { + t.Errorf("the state %s is not the warrants, or ages", raw) } - if !tools[readBy[s.Name]] { - t.Errorf("the state %s has no tool that reads it", s.Name) + if !tools[readBy[name]] { + t.Errorf("the state %s has no tool that reads it", name) } } for _, r := range m.Reads { diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go index 62c4600f..3de340f2 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -4,7 +4,7 @@ package main // mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its // real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered -// from a real nats-server's key-value buckets, mesh/publish onto its subjects). A recorder's hand act is kept in the +// from a real nats-server's key-value buckets, mesh/log.* from the log's real stream, mesh/publish onto its subjects). A recorder's hand act is kept in the // log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once. import ( @@ -185,6 +185,8 @@ func (p *player) handle(m wire) { } case "mesh/subscribe": p.answer(m.ID, map[string]any{}, nil) + case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info": + p.log(m) case "mesh/state.keys": kv, err := p.kv(a.State) if err != nil { @@ -214,6 +216,69 @@ func (p *player) handle(m wire) { } } +// log answers the runtime's log verbs from the log's real stream, LOG__ on mesh.log.... +func (p *player) log(m wire) { + var a struct { + Log string `json:"log"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + From uint64 `json:"from"` + } + _ = json.Unmarshal(m.Params, &a) + stream := "LOG_" + Module + "_" + a.Log + subject := "mesh.log." + Module + "." + a.Log + s, err := p.js.Stream(p.ctx, stream) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entry := func(r *jetstream.RawStreamMsg) map[string]any { + return map[string]any{"sequence": r.Sequence, "key": strings.TrimPrefix(r.Subject, subject+"."), + "value": json.RawMessage(r.Data), "at": r.Time} + } + switch m.Method { + case "mesh/log.append": + ack, err := p.js.Publish(p.ctx, subject+"."+a.Key, a.Value) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"sequence": ack.Sequence}, nil) + case "mesh/log.last": + r, err := s.GetLastMsgForSubject(p.ctx, subject+"."+a.Key) + if errors.Is(err, jetstream.ErrMsgNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, entry(r), nil) + case "mesh/log.read": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entries := []map[string]any{} + for seq := max(a.From, 1); seq <= info.State.LastSeq; seq++ { + if r, err := s.GetMsg(p.ctx, seq); err == nil { + entries = append(entries, entry(r)) + } + } + p.answer(m.ID, map[string]any{"entries": entries, "next": 0, "last": info.State.LastSeq}, nil) + case "mesh/log.info": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"entries": info.State.Msgs, "bytes": info.State.Bytes, + "max_bytes": info.Config.MaxBytes, "first": info.State.FirstSeq, "last": info.State.LastSeq}, nil) + } +} + func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { p.mu.Lock() p.next++ @@ -266,11 +331,18 @@ func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { if err != nil { t.Fatal(err) } - for _, b := range []string{Module + "_" + ActsState, Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} { + for _, b := range []string{Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { t.Fatal(err) } } + // The log as the mesh makes it (ADR 0297): a stream kept by limits, refusing when full, never deleting. + logStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_" + Module + "_" + ActsLog, + Subjects: []string{"mesh.log." + Module + "." + ActsLog + ".>"}, MaxBytes: 64 << 20, Discard: jetstream.DiscardNew, + DenyDelete: true, DenyPurge: true, AllowDirect: true}) + if err != nil { + t.Fatal(err) + } recorded, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + RecordedEvent) if err != nil { t.Fatal(err) @@ -296,10 +368,9 @@ func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" { t.Fatalf("hand-act answered %s", raw) } - acts, _ := js.KeyValue(ctx, Module+"_"+ActsState) - e, err := acts.Get(ctx, r.Act.ID) - if err != nil || !strings.Contains(string(e.Value()), `"by":"ace.mesh-config"`) { - t.Fatalf("the act is not kept on the bus: %v", err) + e, err := logStream.GetLastMsgForSubject(ctx, "mesh.log."+Module+"."+ActsLog+"."+r.Act.ID) + if err != nil || !strings.Contains(string(e.Data), `"by":"ace.mesh-config"`) { + t.Fatalf("the act is not kept in the log on the bus: %v", err) } msg, err := recorded.NextMsg(10 * time.Second) if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) { @@ -317,8 +388,8 @@ func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { t.Fatal(err) } } - if keys, _ := acts.Keys(ctx); len(keys) != 2 { - t.Errorf("the log keeps %v, want the act and one warrant", keys) + if info, err := logStream.Info(ctx); err != nil || info.State.Msgs != 2 { + t.Errorf("the log keeps %v, want the act and one warrant (%v)", info, err) } listed, err := p.call(HandActsTool, "person.jochen", map[string]any{}) if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) { diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json index 3e562b9a..768f3e23 100644 --- a/modules/mesh-hand-acts/module.json +++ b/modules/mesh-hand-acts/module.json @@ -20,15 +20,11 @@ "mesh-controller.hand-acts", "messenger.asks" ], + "logs": [ + "acts" + ], "state": [ - { - "name": "acts", - "ttl-seconds": 34560000 - }, - { - "name": "warrants", - "ttl-seconds": 34560000 - } + "warrants" ], "own-secrets": { "broker": "${dir:state}/broker" From d632461eb9880d6c40762d9c8bb434e7852f282b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:05:37 +0100 Subject: [PATCH 08/20] Add the condition keeper, mesh-conditions, so conditions are raised and cleared from the observers' findings without the controller's lease, held while their module is delivered, and switched only on the operator's take-over (issue 1952 C1-C8, ADR 0564) --- .../cmd/mesh-conditions/compare.go | 218 ++++++ .../cmd/mesh-conditions/hold.go | 308 ++++++++ .../cmd/mesh-conditions/hold_test.go | 109 +++ .../cmd/mesh-conditions/homes.go | 73 ++ .../cmd/mesh-conditions/keeper.go | 660 ++++++++++++++++++ .../cmd/mesh-conditions/keeper_test.go | 306 ++++++++ .../cmd/mesh-conditions/main.go | 161 +++++ .../cmd/mesh-conditions/manifest_test.go | 67 ++ .../cmd/mesh-conditions/reads-outside | 2 + .../cmd/mesh-conditions/tools.go | 252 +++++++ .../mesh-conditions/whole_integration_test.go | 335 +++++++++ modules/mesh-conditions/go.mod | 22 + modules/mesh-conditions/go.sum | 27 + modules/mesh-conditions/module.json | 84 +++ 14 files changed, 2624 insertions(+) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/compare.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/hold.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/hold_test.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/homes.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/keeper.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/main.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/reads-outside create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/tools.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go create mode 100644 modules/mesh-conditions/go.mod create mode 100644 modules/mesh-conditions/go.sum create mode 100644 modules/mesh-conditions/module.json diff --git a/modules/mesh-conditions/cmd/mesh-conditions/compare.go b/modules/mesh-conditions/cmd/mesh-conditions/compare.go new file mode 100644 index 00000000..f2092381 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/compare.go @@ -0,0 +1,218 @@ +package main + +// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare` +// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the +// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and +// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a +// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the +// controller until the lease write, and is refused after it. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Difference is one key the two sides do not agree on, or one listed apart. +type Difference struct { + Key string `json:"key"` + Kind string `json:"kind"` + Reason string `json:"reason"` +} + +// Comparison is compare's answer. +type Comparison struct { + Differences []Difference `json:"differences"` + // Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held. + Apart []Difference `json:"apart"` + Same int `json:"same"` + At time.Time `json:"at"` +} + +// Empty says nothing differs. +func (c Comparison) Empty() bool { return len(c.Differences) == 0 } + +// Compare is the keeper's open conditions against the controller's. +func (k *Keeper) Compare() (Comparison, error) { + mine, err := k.Open() + if err != nil { + return Comparison{}, err + } + k.mu.Lock() + k.init() + theirs := map[string]health.Condition{} + for key, c := range k.ctrl { + theirs[key] = c + } + heldNow := map[string]held{} + for key, h := range k.held { + heldNow[key] = h + } + k.mu.Unlock() + out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()} + open := map[string]health.Condition{} + for _, c := range mine { + open[c.Key] = c + } + keys := map[string]bool{} + for key := range open { + keys[key] = true + } + for key := range theirs { + keys[key] = true + } + for key := range heldNow { + keys[key] = true + } + sorted := make([]string, 0, len(keys)) + for key := range keys { + sorted = append(sorted, key) + } + sort.Strings(sorted) + for _, key := range sorted { + c, inMine := open[key] + t, inTheirs := theirs[key] + kind := c.Kind + if !inMine { + kind = t.Kind + } + home, homed := k.homeOf(kind) + if h, ok := heldNow[key]; ok { + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))}) + continue + } + switch { + case homed && (home.Carried || home.Replaced): + why := "ends with the controller: carried from its bucket until the lease write" + if home.Replaced { + why = "ends with the controller: its fault is observed otherwise" + } + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why}) + case inMine && !inTheirs && (!homed || !home.Controller): + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"}) + case inMine && !inTheirs: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: "open at the keeper, not at the controller"}) + case !inMine && !homed: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)}) + case !inMine: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)}) + case c.Severity != t.Severity: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)}) + default: + out.Same++ + } + } + return out, nil +} + +// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands. +type Asks interface { + Ask(a asks.Ask) error +} + +// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write. +func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a take-over says why") + } + if k.LeaseAway() { + return nil, errors.New("the controller's lease is handed away: the keeper is the one source already") + } + cmp, err := k.Compare() + if err != nil { + return nil, err + } + if !cmp.Empty() { + return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none", + len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason) + } + now := k.Now().UTC() + id := fmt.Sprintf("take-over-%d", now.UnixNano()) + act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id} + bind, err := asks.ActDigest(act) + if err != nil { + return nil, err + } + ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator, + Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+ + "the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+ + "gives up its lease.", cmp.Same), + Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}}, + Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.", + About: Module + "-take-over", Details: "Asked by " + by + ": " + why} + if err := a.Ask(ask); err != nil { + return nil, err + } + return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil +} + +// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper. +func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error { + if o.ID != "yes" { + k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID) + return nil + } + cmp, err := k.Compare() + if err != nil { + return err + } + if !cmp.Empty() { + return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask, + len(cmp.Differences)) + } + return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask}) +} + +// HandBack switches the readers back to the controller, until the lease write. +func (k *Keeper) HandBack(by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a hand-back says why") + } + if k.LeaseAway() { + return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to") + } + t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why} + if err := k.writeTakeover(t); err != nil { + return nil, err + } + return map[string]any{"follow": t.Follow, "at": t.At}, nil +} + +// writeTakeover writes the take-over key, and keeps it in history as the record of the switch. +func (k *Keeper) writeTakeover(t health.Takeover) error { + raw, err := health.EncodeTakeover(t) + if err != nil { + return err + } + _, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return err + } + if found { + _, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev) + } else { + _, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw)) + } + if errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("the take-over key moved under this write: read it again and ask again") + } + if err != nil { + return err + } + k.Takeover(t) + k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By) + return nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go new file mode 100644 index 00000000..8cc8e7b8 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -0,0 +1,308 @@ +package main + +// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12; +// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a +// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or +// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state +// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once, +// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's +// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery +// did not end. A finding about another module or about the machine itself is never held. A condition already open is +// not held: it is no news. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The modules whose delivery holds what they serve beyond their own findings (Limits 12). +const ( + controllerModule = "mesh-controller" + runtimeModule = "node-tools" +) + +// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its +// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names. +var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true} + +// about is a node and a module a finding or a condition is about, for the hold. +type about struct{ Node, Module string } + +// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its +// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is +// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself. +func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about { + var out []about + parts := strings.Split(s.ID, ".") + switch s.Scope { + case health.ScopeModule: + if len(parts) >= 2 { + out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) + } + case health.ScopeProvider: + if len(parts) >= 2 { + out = append(out, about{Node: parts[1], Module: parts[0]}) + } + case health.ScopeSeat: + if len(parts) >= 2 { + out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) + } + case health.ScopeMachine: + // The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is + // delivered. + if source == "S11" || kind == "tools-silent" { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + if s.Machine == "" && len(parts) > 0 { + out[len(out)-1].Node = parts[0] + } + } + return out + } + if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) { + for hk := range k.wants { + if node, module := splitNodeModule(hk); module == controllerModule { + out = append(out, about{Node: node, Module: module}) + } + } + } + if s.Machine != "" && s.Scope != health.ScopeMachine { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + } + return out +} + +func splitNodeModule(hk string) (string, string) { + node, module, _ := strings.Cut(hk, ".") + return node, module +} + +// motion is the delivery of module on node, from its want and have. +func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) { + w, wanted := k.wants[node+"."+module] + h, has := k.haves[node+"."+module] + return delivery.MotionOf(w, wanted, h, has, "", false), w +} + +// holdFor says whether key is held now, and as what. Called with mu held. +func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) { + if open || (f == nil && carried == nil) { + return held{}, false + } + var kind, source string + var subject health.Subject + if f != nil { + kind, source, subject = f.Kind, f.Source, f.Subject + } else { + kind, source, subject = carried.Kind, carried.Source, carried.Subject + } + if _, raisedOut := k.outOfHold[key]; raisedOut { + return held{}, false + } + for _, a := range k.aboutOf(kind, source, subject) { + m, w := k.motion(a.Node, a.Module) + if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP { + continue + } + since := k.Now().UTC() + if was, ok := k.held[key]; ok { + since = was.Since + } + return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true + } + return held{}, false +} + +// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once. +func (k *Keeper) keepHeld(h held) error { + k.mu.Lock() + _, was := k.held[h.Key] + k.held[h.Key] = h + bound := k.bound(h.Module) + k.mu.Unlock() + if was { + return nil + } + if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("keeping %s held: %w", h.Key, err) + } + b := boundKey(h.Node, h.Module) + err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version, + "bound-seconds": int(bound / time.Second)}, bound) + if err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err) + } + k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound) + return nil +} + +// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). The state `bounds` +// keeps each key that long (its ttl-seconds), since the runtime writes no key with a time to live of its own; a bound a +// manifest names needs that first. +func (k *Keeper) bound(string) time.Duration { + if k.Bound > 0 { + return k.Bound + } + return delivery.DefaultDeliveryBound +} + +func boundKey(node, module string) string { return node + "." + module } + +// dropHeld lets a held key go: raised, or gone with why said. +func (k *Keeper) dropHeld(key, why string) { + k.mu.Lock() + h, ok := k.held[key] + delete(k.held, key) + k.mu.Unlock() + if !ok { + return + } + if err := k.Store.Delete(HeldState, key); err != nil { + k.Logf("the held %s could not be removed from held: %v", key, err) + } + if why != "" { + k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why) + } +} + +// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or +// drops what it held. +func (k *Keeper) Want(key string, value []byte, deleted bool) error { + node, module, err := delivery.ParseWantKey(key) + if err != nil { + return nil // not a want of a module on a node (a held fingerprint's key) + } + k.mu.Lock() + k.init() + if deleted { + delete(k.wants, node+"."+module) + } else if w, err := delivery.DecodeWant(value); err == nil { + k.wants[node+"."+module] = w + } else { + k.Logf("the want %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Have takes one change of a node's have. +func (k *Keeper) Have(key string, value []byte, deleted bool) error { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + return nil + } + k.mu.Lock() + k.init() + if deleted { + delete(k.haves, node+"."+module) + } else if h, err := delivery.DecodeHave(value); err == nil { + k.haves[node+"."+module] = h + } else { + k.Logf("the have %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. +func (k *Keeper) BoundPassed(key string) error { + node, module := splitNodeModule(key) + k.mu.Lock() + k.init() + _, w := k.motion(node, module) + holding := false + for _, h := range k.held { + holding = holding || (h.Node == node && h.Module == module) + } + if !holding { + k.mu.Unlock() + return nil + } + k.passed[key] = w.FP + k.mu.Unlock() + return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node, + k.bound(module))) +} + +// judge raises or drops what node and module hold when their delivery ended, or with line past its bound. +func (k *Keeper) judge(node, module, line string) error { + k.mu.Lock() + m, w := k.motion(node, module) + var keys []string + for key, h := range k.held { + if h.Node == node && h.Module == module { + keys = append(keys, key) + } + } + if line == "" && !m.Ended() && m.Delivering { + k.mu.Unlock() + return nil + } + if line == "" { + line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node) + if m.Failed { + line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node) + } + delete(k.passed, node+"."+module) + } + for _, key := range keys { + k.outOfHold[key] = line + } + k.mu.Unlock() + sort.Strings(keys) + var problems []string + for _, key := range keys { + k.dropHeld(key, "") + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + k.mu.Lock() + delete(k.outOfHold, key) + k.mu.Unlock() + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +func short(fp string) string { + if _, rest, ok := strings.Cut(fp, ":"); ok { + fp = rest + } + if len(fp) > 12 { + return fp[:12] + } + return fp +} + +// Held is every key held now, for the read tool and compare. +func (k *Keeper) Held() []held { + k.mu.Lock() + defer k.mu.Unlock() + out := make([]held, 0, len(k.held)) + for _, h := range k.held { + out = append(out, h) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} + +// restoreHeld takes the state `held` as read at the start. +func (k *Keeper) restoreHeld(key string, value []byte) { + var h held + if json.Unmarshal(value, &h) != nil { + return + } + k.mu.Lock() + k.init() + k.held[key] = h + k.mu.Unlock() +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go new file mode 100644 index 00000000..0ee296a2 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves, +// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases +// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented +// order, not a capture; the values are captured. +const ( + ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c" + ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0" + ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6" + ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142" +) + +// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have. +// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still +// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a +// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a +// held key as held. +func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) { + want := func(fp, composed string) []byte { + raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed}) + return raw + } + have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte { + raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h}) + return raw + } + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"}) + for _, c := range []struct { + name string + // end is the have the delivery ends with, or nil for a delivery past its bound. + end []byte + standing bool + said string + line string + }{ + {name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)}, + {name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK), + standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"}, + {name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"}, + {name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true, + said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"}, + } { + r := newRig() + _ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + // The delivery starts: the assigner wants the next fingerprint; the node applies it. + _ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false) + own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now) + other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now) + machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now) + for _, f := range []struct { + key string + raw []byte + }{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} { + if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil { + t.Fatal(err) + } + } + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + if got := r.changes(); got != "raised "+other+", raised "+machine { + t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got) + } + // C8: compare lists the held keys as held, never as a difference. + cmp, _ := r.k.Compare() + held := 0 + for _, d := range cmp.Apart { + if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") { + held++ + } + } + if held != 2 { + t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp) + } + // The carried condition about the controller goes within the delivery; its own finding goes unless it stands. + _ = r.k.Controller("core.controller.deaf", nil, true) + if !c.standing { + _ = r.k.Finding(watchman, own, nil, true) + } + if c.end != nil { + _ = r.k.Have("novox.mesh-controller", c.end, false) + } else { + r.now = r.now.Add(delivery.DefaultDeliveryBound) + _ = r.k.BoundPassed("novox.mesh-controller") + } + if got := r.changes(); got != c.said { + t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said) + } + if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) { + t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation) + } + if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 { + t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held()) + } + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go new file mode 100644 index 00000000..1f0d378c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -0,0 +1,73 @@ +package main + +// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe +// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from +// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the +// lease write, the controller's conditions of the kinds that end with it. +// +// The rows below are what ADR 0564 decides, by the controller's source of each kind as the live controller says it +// (mesh-controller.conditions, read 2026-10-29). **About twenty-five kinds the controller raises have no home in the +// record** (L4.md, "Missing decision"): they have no row here on purpose, and `compare` names each as "no home decided" +// until the lead's decision adds its row. + +// Home is where one condition kind lives. +type Home struct { + // Source is the controller's row or probe that raises it today, for whoever reads the table. + Source string + // Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no + // difference in compare. + Controller bool + // Observer is the module whose findings raise it after the take-over; empty for a kind that ends. + Observer string + // Carried says it ends with the controller's lease and is carried from the controller's bucket until then + // (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller). + Carried bool + // Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in + // compare and not carried. + Replaced bool +} + +// The kinds' homes, by kind. +var Homes = map[string]Home{ + // The watchman's watchdogs (clause 4). + "silent": {Source: "S1", Controller: true, Observer: watchman}, + "max-deliveries": {Source: "S9", Controller: true, Observer: watchman}, + "slow-consumer": {Source: "S9", Controller: true, Observer: watchman}, + "tools-silent": {Source: "S11", Controller: true, Observer: watchman}, + "apply-failing": {Source: "S21", Controller: true, Observer: watchman}, + "module-unhealthy": {Source: "health", Controller: true, Observer: watchman}, + "stuck": {Source: "DM", Controller: true, Observer: watchman}, + // The self-check's probes (clause 6, the Context's table). + "holder-silent": {Source: "D3", Controller: true, Observer: selfCheck}, + "archives-missing": {Source: "D4", Controller: true, Observer: selfCheck}, + "consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck}, + "data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck}, + "agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck}, + "root-not-free": {Source: "D-root", Controller: true, Observer: selfCheck}, + "probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck}, + // The healers' own (clause 7, Limits 2). + "healer-wanted": {Source: "S15", Controller: true, Observer: healers}, + "healers-braked": {Source: "healers", Controller: true, Observer: healers}, + // The kinds that end with the controller and are carried until its lease write (clauses 4, 9). + "controller-deaf": {Source: "S4", Controller: true, Carried: true}, + "lease-lost": {Source: "S12", Controller: true, Carried: true}, + // The kinds that end, their faults observed otherwise (the Context's table). + "core-behind": {Source: "D10", Controller: true, Replaced: true}, +} + +// The observers, as the table names them. +const ( + watchman = "mesh-delivery-watchman" + selfCheck = "mesh-self-check" + healers = "mesh-healers" +) + +// homeOf is the home of a condition's kind, and whether the table has one. +func (k *Keeper) homeOf(kind string) (Home, bool) { + homes := k.Homes + if homes == nil { + homes = Homes + } + h, ok := homes[kind] + return h, ok +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go new file mode 100644 index 00000000..70a82f3d --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -0,0 +1,660 @@ +package main + +// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the +// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and, +// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From +// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set, +// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition +// but its finding going (ADR 0231 §14). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The module, its seat, and the states it keeps. +const ( + Module = health.KeeperModule + Seat = "mesh-conditions" + + ConditionsState = health.ConditionsState + TakeoverState = health.TakeoverState + HistoryState = "history" + HeldState = "held" + BoundsState = "bounds" + // AskedState keeps the keeper's asks of the operator (go/asker). + AskedState = "asked" +) + +// HistoryKeptFor is how long a transition is kept (clause 3). +const HistoryKeptFor = 90 * 24 * time.Hour + +// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's). +var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule} + +// The other states it reads, as `.`. +const ( + AttemptsRead = health.HealersModule + "." + health.AttemptsState + ControllerConditions = "mesh-controller.conditions" + ControllerLease = "mesh-controller.lease" + ControllerLeaseKey = "holder" + WantsRead = delivery.AssignerModule + ".wants" + HavesRead = "node-engine.have" + ConditionWrites = 8 +) + +// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to +// live for a hold's bound. +type Store interface { + Get(state, key string) (value []byte, revision uint64, found bool, err error) + Create(state, key string, value any) (uint64, error) + Update(state, key string, value any, revision uint64) (uint64, error) + Delete(state, key string) error + Keys(state string) ([]string, error) + CreateTTL(state, key string, value any, ttl time.Duration) error +} + +// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)). +type held struct { + Key string `json:"key"` + Node string `json:"node"` + Module string `json:"module"` + Since time.Time `json:"since"` + Version string `json:"version"` + Wanted json.RawMessage `json:"-"` +} + +// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening. +type clearedMemo struct { + at time.Time + count int + began time.Time + gaps []health.Gap + silenced *health.Silence + tried []health.Tried +} + +// Keeper is the condition keeper's state and its writes. +type Keeper struct { + Store Store + // Say publishes one event on the seat; History keeps one transition. + Say func(e health.Event) error + Now func() time.Time + Logf func(format string, a ...any) + // Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil. + Homes map[string]Home + // Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go). + Bound time.Duration + + mu sync.Mutex + findings map[string]map[string]health.Finding + blind map[string]string + attempts map[string]health.Attempt + ctrl map[string]health.Condition + leaseOut bool + takeover health.Takeover + wants map[string]delivery.Want + haves map[string]delivery.Have + stuck map[string]string + held map[string]held + // passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the + // line a key raised out of a hold says (hold.go). + passed map[string]string + outOfHold map[string]string + cleared map[string]clearedMemo + seq uint64 +} + +func (k *Keeper) init() { + if k.findings == nil { + k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{}, + map[string]health.Attempt{}, map[string]health.Condition{} + k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{}, + map[string]string{}, map[string]held{}, map[string]clearedMemo{} + k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController} + k.passed, k.outOfHold = map[string]string{}, map[string]string{} + } + if k.Now == nil { + k.Now = time.Now + } + if k.Logf == nil { + k.Logf = func(string, ...any) {} + } +} + +// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears +// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was. +func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if k.findings[observer] == nil { + k.findings[observer] = map[string]health.Finding{} + } + if deleted { + delete(k.findings[observer], key) + } else { + f, err := health.DecodeFinding(value) + if err != nil { + k.mu.Unlock() + k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err) + return nil + } + if f.Key != key { + k.mu.Unlock() + k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key) + return nil + } + k.findings[observer][key] = f + } + k.mu.Unlock() + return k.Settle(key) +} + +// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing +// (ADR 0227 §4). Empty why says it is read again. +func (k *Keeper) Blind(observer, why string) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + if why == "" { + delete(k.blind, observer) + } else { + k.blind[observer] = why + } +} + +// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation +// (clause 7). +func (k *Keeper) Attempt(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + var condition string + if old, ok := k.attempts[key]; ok { + condition = old.Condition + } + if deleted { + delete(k.attempts, key) + } else { + var a health.Attempt + if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" { + k.mu.Unlock() + k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err) + return nil + } + k.attempts[key], condition = a, a.Condition + } + k.mu.Unlock() + if condition == "" { + return nil + } + return k.Settle(condition) +} + +// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write. +func (k *Keeper) Controller(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if deleted { + delete(k.ctrl, key) + } else { + var c health.Condition + if err := json.Unmarshal(value, &c); err != nil { + k.mu.Unlock() + k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err) + return nil + } + k.ctrl[key] = c + } + k.mu.Unlock() + return k.Settle(key) +} + +// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no +// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear. +func (k *Keeper) Lease(away bool) error { + k.mu.Lock() + k.init() + was := k.leaseOut + k.leaseOut = away + keys := make([]string, 0, len(k.ctrl)) + for key := range k.ctrl { + keys = append(keys, key) + } + k.mu.Unlock() + if was == away { + return nil + } + return k.settleAll(keys) +} + +// LeaseAway says whether the controller's lease is handed away. +func (k *Keeper) LeaseAway() bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.leaseOut +} + +// Takeover takes the keeper's own take-over key as read. +func (k *Keeper) Takeover(t health.Takeover) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + k.takeover = t +} + +// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves +// it under this one. +func (k *Keeper) Settle(key string) error { + var err error + for i := 0; i < ConditionWrites; i++ { + if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err) +} + +func (k *Keeper) settleAll(keys []string) error { + sort.Strings(keys) + var problems []string + for _, key := range keys { + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's +// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it. +func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) { + observers := make([]string, 0, len(k.findings)) + for o := range k.findings { + observers = append(observers, o) + } + sort.Strings(observers) + for _, o := range observers { + if f, ok := k.findings[o][key]; ok { + return &f, nil + } + } + if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) { + return nil, &c + } + return nil, nil +} + +func (k *Keeper) endsWithController(c health.Condition) bool { + h, ok := k.homeOf(c.Kind) + return ok && h.Carried +} + +// settleOnce is one try of Settle. +func (k *Keeper) settleOnce(key string) error { + now := k.Now().UTC() + raw, rev, open, err := k.Store.Get(ConditionsState, key) + if err != nil { + return fmt.Errorf("reading the condition %s: %w", key, err) + } + var was health.Condition + if open { + if err := json.Unmarshal(raw, &was); err != nil { + return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + } + k.mu.Lock() + k.init() + f, carried := k.wanted(key) + hold, holding := k.holdFor(key, f, carried, open) + k.mu.Unlock() + + if f == nil && carried == nil { + k.dropHeld(key, "its finding went while its module was delivered") + if !open { + return nil + } + return k.clear(key, was, rev, now, "its finding went") + } + if holding { + return k.keepHeld(hold) + } + k.dropHeld(key, "") + next := k.derive(key, was, open, f, carried, now) + events := transitions(was, next, open) + if len(events) == 0 && open && sameBody(was, next) { + return nil + } + if open { + _, err = k.Store.Update(ConditionsState, key, next, rev) + } else { + _, err = k.Store.Create(ConditionsState, key, next) + } + if err != nil { + return err + } + k.mu.Lock() + delete(k.cleared, key) + k.mu.Unlock() + for _, e := range events { + e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change) + e.Show = k.show(next, carried != nil) + k.tell(e) + } + return nil +} + +func (k *Keeper) show(c health.Condition, carried bool) string { + if carried { + return health.ShowController(c.Key) + } + return health.ShowKeeper(c.Key) +} + +// derive is the condition key should be now, from what was open and what the inputs say. +func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition { + var c health.Condition + switch { + case carried != nil: + // Raised and cleared as the controller's bucket says, with its show and actions (Limits 9). + c = *carried + c.Epoch = 0 + c.LooksUnread = "" + case open: + c = was + k.fromFinding(&c, *f, now) + default: + c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf} + k.fromFinding(&c, *f, now) + k.mu.Lock() + if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin { + c.Count = before.count + 1 + c.First = before.began + c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now}) + if over := len(c.Gaps) - health.KeptGaps; over > 0 { + c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:] + } + if before.silenced != nil && now.Before(before.silenced.Until) { + c.Silenced = before.silenced + } + c.Tried = before.tried + } + k.mu.Unlock() + } + if carried == nil { + k.overlay(&c, now) + } + if !open { + k.mu.Lock() + if line := k.outOfHold[key]; line != "" { + c.Explanation = strings.TrimSpace(c.Explanation + " " + line) + } + k.mu.Unlock() + } + if c.Silenced != nil && !now.Before(c.Silenced.Until) { + c.Silenced = nil + } + return c +} + +// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence +// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again. +func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) { + escalated := c.Escalated() + c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source + c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions + if !(escalated && f.Severity != health.Urgent) { + c.Severity = f.Severity + } + if !escalated { + c.Resolver = f.Resolver + if c.Resolver == "" { + c.Resolver = health.ResolverSelf + } + } + if len(f.Evidence) > 0 { + c.Evidence = append([]health.Evidence(nil), f.Evidence...) + c.LastObserved = f.Evidence[0].At + } else if c.Evidence == nil { + c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}} + c.LastObserved = f.At + } + c.Observations = len(c.Evidence) + if c.Actions == nil { + c.Actions = []health.Action{} + } + _ = now +} + +// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's +// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has +// none of its own. +func (k *Keeper) overlay(c *health.Condition, now time.Time) { + k.mu.Lock() + var mine []health.Attempt + for _, a := range k.attempts { + if a.Condition == c.Key { + mine = append(mine, a) + } + } + ctrl, carry := k.ctrl[c.Key] + carry = carry && !k.leaseOut + k.mu.Unlock() + sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) }) + if carry && len(ctrl.Tried) > 0 { + c.Tried = append([]health.Tried(nil), ctrl.Tried...) + if ctrl.Escalated() { + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + } else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf { + c.Resolver = ctrl.Resolver + } + } + for _, a := range mine { + t := a.Tried() + if !containsTried(c.Tried, t) { + c.Tried = append(c.Tried, t) + } + switch a.Outcome { + case health.OutcomeEscalated: + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + default: + if c.Resolver != health.ResolverOperator { + c.Resolver = health.ResolverHealer(a.Healer) + } + } + } + if len(c.Tried) > health.KeptTried { + c.Tried = c.Tried[len(c.Tried)-health.KeptTried:] + } + if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) { + c.Silenced = ctrl.Silenced + } +} + +func containsTried(list []health.Tried, t health.Tried) bool { + for _, x := range list { + if x.At.Equal(t.At) && x.By == t.By && x.What == t.What { + return true + } + } + return false +} + +// transitions are the events between what was open and what is now, in the controller's order: raised or reopened; +// else severity, resolver, silence and words changes. An observation that changes none of these says nothing. +func transitions(was, now health.Condition, open bool) []health.Event { + if !open { + change := health.ChangeRaised + if now.Count > 1 { + change = health.ChangeReopened + } + return []health.Event{{Change: change}} + } + var out []health.Event + if was.Severity != now.Severity { + out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)}) + } + if was.Resolver != now.Resolver { + out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver}) + } + switch { + case was.Silenced == nil && now.Silenced != nil: + out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why}) + case was.Silenced != nil && now.Silenced == nil: + out = append(out, health.Event{Change: health.ChangeUnsilenced}) + } + if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) { + out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs}) + } + return out +} + +func sameActions(a, b []health.Action) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) || (len(a) == 0 && len(b) == 0) +} + +func sameBody(a, b health.Condition) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// clear removes an open condition whose finding went, and says it (clause 2). +func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error { + // A delete at the revision read: a write that moved it meanwhile is read again. + if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil { + return err + } + if err := k.Store.Delete(ConditionsState, key); err != nil { + return err + } + k.mu.Lock() + k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried} + k.mu.Unlock() + cleared := now + k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why, + Cleared: &cleared, Show: k.show(was, false)}) + return nil +} + +// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition +// and its history still hold it. +func (k *Keeper) tell(e health.Event) { + k.mu.Lock() + k.seq++ + hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq) + k.mu.Unlock() + if _, err := k.Store.Create(HistoryState, hk, e); err != nil { + k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err) + } + if k.Say == nil { + return + } + if err := k.Say(e); err != nil { + k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err) + } +} + +// Open is every open condition, ordered as `conditions` says them. +func (k *Keeper) Open() ([]health.Condition, error) { + keys, err := k.Store.Keys(ConditionsState) + if err != nil { + return nil, err + } + var out []health.Condition + for _, key := range keys { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + continue + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + out = append(out, c) + } + health.Order(out) + return out, nil +} + +// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared. +func (k *Keeper) Resync() (int, error) { + open, err := k.Open() + if err != nil { + return 0, err + } + seen := map[string]bool{} + for _, c := range open { + seen[c.Key] = true + } + k.mu.Lock() + k.init() + for o, fs := range k.findings { + if k.blind[o] != "" { + continue + } + for key := range fs { + seen[key] = true + } + } + for key := range k.ctrl { + seen[key] = true + } + k.mu.Unlock() + keys := make([]string, 0, len(seen)) + for key := range seen { + keys = append(keys, key) + } + return len(keys), k.settleAll(keys) +} + +// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in +// history, naming who and why. +func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) { + if strings.TrimSpace(why) == "" { + return health.Condition{}, errors.New("a silence says why") + } + if d <= 0 || d > health.MaxSilence { + return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d) + } + for i := 0; i < ConditionWrites; i++ { + raw, rev, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return health.Condition{}, err + } + if !found || string(raw) == "null" { + return health.Condition{}, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return health.Condition{}, err + } + now := k.Now().UTC() + c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now} + if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) { + continue + } else if err != nil { + return health.Condition{}, err + } + k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced, + Why: why + " (" + by + ")", Show: health.ShowKeeper(key)}) + return c, nil + } + return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key) +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go new file mode 100644 index 00000000..6f8dd1d8 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// fakeStore is the bus's states in memory, with revisions, as the runtime answers them. +type fakeStore struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + next uint64 +} + +func newFakeStore() *fakeStore { + return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}} +} + +func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.vals[state+"/"+key] + return v, f.revs[state+"/"+key], ok, nil +} + +func (f *fakeStore) put(state, key string, value any) uint64 { + raw, _ := json.Marshal(value) + f.next++ + f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next + return f.next +} + +func (f *fakeStore) Create(state, key string, value any) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.vals[state+"/"+key]; ok { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.revs[state+"/"+key] != rev { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Delete(state, key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.vals, state+"/"+key) + delete(f.revs, state+"/"+key) + return nil +} + +func (f *fakeStore) Keys(state string) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.vals { + if s, key, _ := strings.Cut(k, "/"); s == state { + out = append(out, key) + } + } + sort.Strings(out) + return out, nil +} + +func (f *fakeStore) CreateTTL(state, key string, value any, _ time.Duration) error { + _, err := f.Create(state, key, value) + return err +} + +// rig is a keeper over a fake store, its clock and what it said. +type rig struct { + k *Keeper + st *fakeStore + now time.Time + said []health.Event +} + +func newRig() *rig { + r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)} + r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error { + r.said = append(r.said, e) + return nil + }} + return r +} + +func (r *rig) changes() string { + var out []string + for _, e := range r.said { + out = append(out, e.Change+" "+e.Key) + } + r.said = nil + return strings.Join(out, ", ") +} + +func (r *rig) open(key string) (health.Condition, bool) { + raw, _, ok, _ := r.st.Get(ConditionsState, key) + var c health.Condition + if ok { + _ = json.Unmarshal(raw, &c) + } + return c, ok +} + +// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding). +func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) { + t.Helper() + f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id}, + Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source, + Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + return f.Key, raw +} + +// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer +// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten +// minutes is the same fault; a silence is a transition in history. +func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) { + r := newRig() + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + steps := []struct { + name string + do func() error + said string + check func(c health.Condition, open bool) string + }{ + {name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) }, + said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string { + if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }}, + {name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err }, + check: func(c health.Condition, open bool) string { + if !open { + return "cleared by blindness" + } + return "" + }}, + {name: "a healer's attempt reaches tried and the resolver", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now}) + return r.k.Attempt("h3-1", a, false) + }, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "an escalation makes it urgent, the operator's", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated, + Begun: r.now.Add(time.Minute)}) + return r.k.Attempt("h3-2", a, false) + }, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "a silence is a transition with who and why", do: func() error { + _, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable") + return err + }, said: "silenced machine.g14.silent"}, + {name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) }, + said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string { + if open { + return "still open" + } + return "" + }}, + {name: "raised again within ten minutes is a reopening", do: func() error { + r.now = r.now.Add(5 * time.Minute) + return r.k.Finding(watchman, key, raw, false) + }, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Count != 2 || len(c.Gaps) != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + // Until the lease write the controller's escalation for the same key reaches the condition (clause 8). + {name: "the controller's escalation is carried", do: func() error { + ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator, + Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}}) + _ = r.k.Attempt("h3-1", nil, true) + _ = r.k.Attempt("h3-2", nil, true) + return r.k.Controller(key, ctrl, false) + }, check: func(c health.Condition, _ bool) string { + if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + } + for _, s := range steps { + if err := s.do(); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" { + t.Errorf("%s: said %q, want %q", s.name, got, s.said) + } + if s.check != nil { + c, open := r.open(key) + if why := s.check(c, open); why != "" { + t.Errorf("%s: %s", s.name, why) + } + } + } + // Every transition is kept in history, and each event carries the keeper's show. + keys, _ := r.st.Keys(HistoryState) + if len(keys) < 7 { + t.Errorf("history keeps %d transitions", len(keys)) + } + raw0, _, _, _ := r.st.Get(HistoryState, keys[0]) + if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) { + t.Errorf("the first transition is %s", raw0) + } + // A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the + // lease is handed away, with the controller's show (clause 9, Limits 9). + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9}) + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" { + t.Errorf("a carried kind is %+v, said %+v", c, r.said) + } + r.said = nil + if err := r.k.Lease(true); err != nil { + t.Fatal(err) + } + if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" { + t.Error("a carried kind outlived the lease write") + } +} + +// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held +// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed +// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and +// hand-back after the lease write. +func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) { + r := newRig() + put := func(key, kind string) []byte { + raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"}) + return raw + } + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + _ = r.k.Finding(watchman, key, raw, false) + _ = r.k.Controller(key, put(key, "silent"), false) + only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now) + _ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false) + _ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false) + _ = r.k.Controller("delivery.x.stalled", put("delivery.x.stalled", "stalled"), false) + _ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false) + cmp, err := r.k.Compare() + if err != nil { + t.Fatal(err) + } + reasons := map[string]string{} + for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) { + reasons[d.Key] = d.Reason + } + for key, says := range map[string]string{ + only: "keeper only", + "core.controller.deaf": "carried", + "delivery.x.stalled": "no home is decided for the kind stalled", + "seat.vault.ace.holder-silent": "mesh-self-check has not written its finding", + } { + if !strings.Contains(reasons[key], says) { + t.Errorf("%s: %q, want it to say %q", key, reasons[key], says) + } + } + if cmp.Same != 1 || len(cmp.Differences) != 2 { + t.Errorf("compare: %+v", cmp) + } + if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") { + t.Errorf("take-over while compare differs: %v", err) + } + if _, err := r.k.HandBack("jochen", "back"); err != nil { + t.Errorf("hand-back before the lease write: %v", err) + } + _ = r.k.Lease(true) + if _, err := r.k.HandBack("jochen", "back"); err == nil { + t.Error("hand-back after the lease write was taken") + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go new file mode 100644 index 00000000..2a1c654e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -0,0 +1,161 @@ +// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A +// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its +// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the +// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's +// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the +// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP +// channel; what this module says, it says on stderr. +package main + +import ( + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} + +func (runtimeStore) Create(state, key string, value any) (uint64, error) { + return stdio.State(state).Create(key, value) +} + +func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) { + return stdio.State(state).Update(key, value, revision) +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// CreateTTL creates the key in a state whose own age is the time to live (`bounds`, ttl-seconds): the runtime writes no +// key with a time to live of its own. +func (runtimeStore) CreateTTL(state, key string, value any, _ time.Duration) error { + _, err := stdio.State(state).Create(key, value) + return err +} + +// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3). +func sayOnSeat(e health.Event) error { + return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "") +} + +// NewKeeper is the keeper over the runtime. +func NewKeeper() *Keeper { + return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf} +} + +// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not +// delivered) is read again when it is. +func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) { + go func() { + for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) { + err := stdio.State(state).Watch("", handle) + if err == nil { + if observer != "" { + k.Blind(observer, "") + } + return + } + if observer != "" { + k.Blind(observer, err.Error()) + } + logf("%s cannot be watched (%v); watched again in %s", state, err, wait) + time.Sleep(wait) + } + }() +} + +// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13). +func Keep(k *Keeper) { + for _, s := range []string{HeldState} { + keys, _ := stdio.State(s).Keys() + for _, key := range keys { + if e, err := stdio.State(s).Get(key); err == nil && e != nil { + k.restoreHeld(key, e.Value) + } + } + } + if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil { + var raw []byte + if e != nil { + raw = e.Value + } + if t, err := health.DecodeTakeover(raw, e != nil); err == nil { + k.Takeover(t) + } + } + said := func(err error) error { + if err != nil { + logf("%v", err) + } + return nil + } + watch(k, ControllerLease, "", func(c stdio.StateChange) error { + if c.Key != ControllerLeaseKey { + return nil + } + away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil) + logf("the controller's lease: %s", why) + return said(k.Lease(away)) + }) + watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) }) + watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) }) + watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error { + return said(k.Attempt(c.Key, c.Value, c.Op != "put")) + }) + watch(k, ControllerConditions, "", func(c stdio.StateChange) error { + if k.LeaseAway() { + return nil + } + return said(k.Controller(c.Key, c.Value, c.Op != "put")) + }) + for _, o := range Observers { + watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error { + return said(k.Finding(o, c.Key, c.Value, c.Op != "put")) + }) + } + watch(k, BoundsState, "", func(c stdio.StateChange) error { + if c.Op != "put" && !c.Current { + return said(k.BoundPassed(c.Key)) + } + return nil + }) +} + +func main() { + k := NewKeeper() + client := asker.New(AskedState) + client.Module = Module + go func() { + <-stdio.Served() + Keep(k) + if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) { + if err != nil { + logf("the answer to %s (%s): %v", w.Ask, h, err) + } + }); err != nil { + logf("the operator's answers cannot be heard: %v", err) + } + }() + if err := stdio.Serve("", Tools(k, client)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go new file mode 100644 index 00000000..66174fa2 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat +// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers' +// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its +// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it, +// every act a tool that starts it. +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("the manifest is refused: %v", err) + } + seat, _ := catalogue.SeatNamed(Seat) + if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat || + !slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) { + t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves)) + } + reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead} + for _, o := range Observers { + reads = append(reads, o+"."+health.FindingsState) + } + slices.Sort(reads) + got := slices.Clone(m.Reads) + slices.Sort(got) + if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) { + t.Errorf("reads %v, want %v and never the healers' would", got, reads) + } + ttl := map[string]int{} + for _, s := range m.State { + ttl[s.Name] = s.TTLSeconds + } + if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 900 { + t.Errorf("history is kept %d s and a bound %d s", ttl[HistoryState], ttl[BoundsState]) + } + // Clause 12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool, + HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync", + "what it reads": InputsTool} + tools := map[string]bool{} + for _, tool := range Tools(&Keeper{}, nil) { + tools[tool.Name] = true + } + for name := range ttl { + if readBy[name] == "" { + t.Errorf("the state %s has no tool that reads it", name) + } + } + for what, tool := range readBy { + if !tools[tool] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/reads-outside b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside new file mode 100644 index 00000000..c20a67a5 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-conditions/module.json diff --git a/modules/mesh-conditions/cmd/mesh-conditions/tools.go b/modules/mesh-conditions/cmd/mesh-conditions/tools.go new file mode 100644 index 00000000..330e153e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/tools.go @@ -0,0 +1,252 @@ +package main + +// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions` +// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the +// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call. + +import ( + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"} + +// The keeper's own read tools. +const ( + HistoryTool = "conditions_history" + HeldTool = "conditions_held" + TakeoverTool = "conditions_takeover" + InputsTool = "conditions_inputs" +) + +// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314). +const briefEvidence = 1 + +// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole. +type inBrief struct { + health.Condition + EvidenceKept int `json:"evidence-kept,omitempty"` + More string `json:"more,omitempty"` +} + +func obj(props map[string]any, required ...string) map[string]any { + out := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + out["required"] = required + } + return out +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +// Tools are what the runtime serves. +func Tools(k *Keeper, a Asks) []stdio.Tool { + return []stdio.Tool{ + {Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " + + "oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " + + "one whole; with history, every transition lately.", + Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"), + "severity": str("urgent or warning"), "machine": str("only those about this machine"), + "history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}), + Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }}, + {Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " + + "transition in history; it stays open.", + Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"), + "why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"), + Run: func(args map[string]any) (any, error) { + d, err := duration(strArg(args, "for")) + if err != nil { + return nil, err + } + return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " + + "its reason; what ends with the controller, what only the keeper raises and what is held listed apart.", + Run: func(map[string]any) (any, error) { return k.Compare() }}, + {Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " + + "keeper; refused while compare differs.", + Input: obj(map[string]any{"why": str("why now")}, "why"), + Run: func(args map[string]any) (any, error) { + return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " + + "is handed away.", + Input: obj(map[string]any{"why": str("why")}, "why"), + Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }}, + {Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.", + Run: func(map[string]any) (any, error) { + n, err := k.Resync() + return map[string]any{"settled": n}, err + }}, + {Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.", + Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}), + Run: func(args map[string]any) (any, error) { + return k.History(strArg(args, "key"), intArg(args, "limit", 50)) + }}, + {Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " + + "with its module, node, version and since when, and the deliveries' bounds.", + Run: func(map[string]any) (any, error) { + bounds, err := k.Store.Keys(BoundsState) + return map[string]any{"held": k.Held(), "bounds": bounds}, err + }}, + {Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.", + Run: func(map[string]any) (any, error) { + raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return nil, err + } + return health.DecodeTakeover(raw, found) + }}, + {Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " + + "cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.", + Run: func(map[string]any) (any, error) { return k.Inputs(), nil }}, + } +} + +// ConditionsVerb is `conditions`: the list, one whole, or the history. +func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) { + key := strArg(args, "key") + if strArg(args, "history") == "true" { + days := intArg(args, "days", 7) + if days < 1 || days > 90 { + return nil, fmt.Errorf("days is from 1 to 90, not %d", days) + } + return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days) + } + if key != "" { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + return nil, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + return c, json.Unmarshal(raw, &c) + } + scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine") + if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) { + return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity) + } + open, err := k.Open() + if err != nil { + return nil, err + } + list := []inBrief{} + counted := map[string]int{} + for _, c := range open { + if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) || + (machine != "" && !concerns(c, machine)) { + continue + } + b := inBrief{Condition: c} + if len(c.Evidence) > briefEvidence { + b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key) + } + list = append(list, b) + counted[c.Source+" "+c.Kind]++ + } + return map[string]any{"conditions": list, "open": len(open), "counted": counted, + "note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " + + "its newest evidence — `conditions key=` gives one whole"}, nil +} + +// concerns says whether a condition is about a machine: it names it, or its key does. +func concerns(c health.Condition, machine string) bool { + return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || + slices.Contains(strings.Split(c.Key, "."), machine) +} + +// History is the newest transitions, at most limit, of one key or all. +func (k *Keeper) History(key string, limit int) ([]health.Event, error) { + all, err := k.HistorySince(key, time.Time{}, 0) + if err != nil { + return nil, err + } + events := all["history"].([]health.Event) + slices.Reverse(events) + if len(events) > limit { + events = events[:limit] + } + return events, nil +} + +// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it. +func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) { + keys, err := k.Store.Keys(HistoryState) + if err != nil { + return nil, err + } + events := []health.Event{} + for _, hk := range keys { + raw, _, found, err := k.Store.Get(HistoryState, hk) + if err != nil || !found { + continue + } + var e health.Event + if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) { + continue + } + events = append(events, e) + } + return map[string]any{"history": events, "days": days}, nil +} + +// Inputs is what the keeper reads, as it stands. +func (k *Keeper) Inputs() map[string]any { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + findings := map[string]int{} + for o, fs := range k.findings { + findings[o] = len(fs) + } + return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts), + "controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow, + "wants": len(k.wants), "haves": len(k.haves)} +} + +func strArg(a map[string]any, name string) string { + switch v := a[name].(type) { + case string: + return strings.TrimSpace(v) + case bool: + return strconv.FormatBool(v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + } + return "" +} + +func intArg(a map[string]any, name string, def int) int { + if n, err := strconv.Atoi(strArg(a, name)); err == nil { + return n + } + return def +} + +// duration reads 30m, 4h or 2d. +func duration(s string) (time.Duration, error) { + if n, ok := strings.CutSuffix(s, "d"); ok { + days, err := strconv.Atoi(n) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return time.Duration(days) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return d, nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go new file mode 100644 index 00000000..877a8ce1 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -0,0 +1,335 @@ +//go:build integration + +package main + +// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary, +// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a +// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its +// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as +// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as +// condition-cleared. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the keeper needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is a state's bucket: the keeper's own by its local name, another's by `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, read := strings.Cut(state, "."); read { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + kv, kvErr := p.kv(a.State) + switch m.Method { + case "mesh/state.get": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.create", "mesh/state.update", "mesh/state.put": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + var rev uint64 + var err error + switch m.Method { + case "mesh/state.create": + rev, err = kv.Create(p.ctx, a.Key, a.Value) + case "mesh/state.update": + rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision) + default: + rev, err = kv.Put(p.ctx, a.Key, a.Value) + } + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}}) + return + } + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.keys": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + err := kvErr + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watchID string + _ = json.Unmarshal(m.ID, &watchID) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID, + "change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())), + "revision": e.Revision(), "current": current}}}) + } + case "mesh/seat.publish": + p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body)) + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func nullIfEmpty(b []byte) []byte { + if len(b) == 0 { + return []byte("null") + } + return b +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears +// it. +func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the keeper does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", + "mesh-healers_attempts"} + for _, o := range Observers { + buckets = append(buckets, o+"_findings") + } + for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} { + buckets = append(buckets, Module+"_"+s) + } + for _, b := range buckets { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>") + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + // The first call serves; the watches start after it. + if _, err := p.call(InputsTool, map[string]any{}); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + at := time.Now().UTC() + f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14", + Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline", + Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1", + Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings") + if _, err := findings.Put(ctx, f.Key, raw); err != nil { + t.Fatal(err) + } + msg, err := events.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("no condition-raised was said: %v", err) + } + var e health.Event + if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" || + e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline { + t.Fatalf("said %s on %s", msg.Data, msg.Subject) + } + listed, err := p.call(Seat+".conditions", map[string]any{}) + if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) { + t.Fatalf("conditions answered %s, %v", listed, err) + } + if err := findings.Delete(ctx, f.Key); err != nil { + t.Fatal(err) + } + msg, err = events.NextMsg(10 * time.Second) + if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" { + t.Fatalf("no condition-cleared was said: %v", err) + } + conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState) + if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the cleared condition is still kept: %v", err) + } +} diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod new file mode 100644 index 00000000..fcfe14e1 --- /dev/null +++ b/modules/mesh-conditions/go.mod @@ -0,0 +1,22 @@ +module mesh-conditions + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-conditions/go.sum b/modules/mesh-conditions/go.sum new file mode 100644 index 00000000..8a5c3b7a --- /dev/null +++ b/modules/mesh-conditions/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 h1:+jH77IJmxUiFDdbJljhynQRY0Cc5/5jtwmMV7UUqRI4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json new file mode 100644 index 00000000..20805f2b --- /dev/null +++ b/modules/mesh-conditions/module.json @@ -0,0 +1,84 @@ +{ + "module": "mesh-conditions", + "version": "1", + "runs-as": "mesh-conditions", + "claims": [ + { + "name": "mesh-conditions", + "scope": "mesh", + "serves": [ + "conditions", + "silence", + "compare", + "take-over", + "hand-back", + "resync" + ] + } + ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-watchman.findings", + "mesh-self-check.findings", + "mesh-healers.findings", + "mesh-delivery-assigner.findings", + "mesh-healers.attempts", + "mesh-controller.conditions", + "mesh-controller.lease", + "mesh-delivery-assigner.wants", + "node-engine.have" + ], + "state": [ + "conditions", + { + "name": "history", + "ttl-seconds": 7776000 + }, + "takeover", + "held", + { + "name": "bounds", + "ttl-seconds": 900, + "key-ttl": true + }, + "asked" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-conditions", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-conditions", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-conditions" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-conditions", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-conditions", + "binary": "mesh-conditions", + "loads": [ + "mesh-conditions" + ] + } + ] + } +} From b5fdcb88c8de72e6c73ab51f437f0afa1ed047e3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:12:08 +0100 Subject: [PATCH 09/20] Say checked, healer-acted and act-recorded on their seats' subjects, where their consumers hear them, and pin mesh-sdk fbd00c8 (issues 1952, 1953) stdio.Emit put them on the module's subject, which no consumer of a seat's event hears. The manifests no longer list a seat's events as the module's own, which the SDK now refuses. mesh-hand-acts takes its act layout and names from mesh-sdk's handacts, and counts a repeated cause by its Repair rule. --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 +- .../mesh-hand-acts/cmd/mesh-hand-acts/acts.go | 106 +++++------------- .../cmd/mesh-hand-acts/acts_test.go | 4 +- .../mesh-hand-acts/cmd/mesh-hand-acts/main.go | 2 +- .../cmd/mesh-hand-acts/manifest_test.go | 3 +- .../mesh-hand-acts/whole_integration_test.go | 9 +- modules/mesh-hand-acts/go.mod | 2 +- modules/mesh-hand-acts/go.sum | 4 +- modules/mesh-hand-acts/module.json | 3 - .../mesh-healers/cmd/mesh-healers/healers.go | 13 ++- .../cmd/mesh-healers/healers_test.go | 4 +- modules/mesh-healers/cmd/mesh-healers/main.go | 6 +- .../cmd/mesh-healers/manifest_test.go | 2 +- .../mesh-healers/whole_integration_test.go | 9 +- modules/mesh-healers/go.mod | 2 +- modules/mesh-healers/go.sum | 4 +- modules/mesh-healers/module.json | 3 - .../cmd/mesh-self-check/main.go | 4 +- .../cmd/mesh-self-check/manifest_test.go | 3 +- .../cmd/mesh-self-check/selfcheck.go | 14 ++- .../cmd/mesh-self-check/selfcheck_test.go | 11 +- .../mesh-self-check/whole_integration_test.go | 9 +- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +- modules/mesh-self-check/module.json | 3 - 26 files changed, 102 insertions(+), 130 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index 30f01e1d..68e8f5fc 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c diff --git a/checks/health/go.sum b/checks/health/go.sum index aacdf9f6..d598087e 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go index 01e09440..6e8579e8 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go @@ -25,62 +25,35 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/handacts" ) -// Module and Seat are this module and the seat it holds once for the mesh. +// The seat, the log, the states and the event, as mesh-sdk's handacts names them. const ( - Module = "mesh-hand-acts" - Seat = "mesh-hand-acts" -) - -// The states it keeps and reads, and the event it says. -const ( - ActsLog = "acts" - WarrantsState = "warrants" - // ControllerActs is the controller's log, read until its lease ends. - ControllerActs = "mesh-controller.hand-acts" + Module = handacts.Module + Seat = handacts.Seat + ActsLog = handacts.ActsLog + WarrantsState = handacts.WarrantsState + ControllerActs = handacts.ControllerActs // RouterRecords is the router's record of every ask, `.` (ADR 0259): the warrant is read there. RouterRecords = "messenger.asks" - RecordedEvent = "act-recorded" -) + RecordedEvent = handacts.EventRecorded + // RecordedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + RecordedSubject = "mesh.seat." + Seat + ".event." + RecordedEvent -// The verbs an act is recorded under, as the controller's log names them: what S15 reads to tell a repair from a -// person's decision by design. -const ( - VerbRecord = "hand-act record" - VerbDrill = "hand-act drill" - VerbWarrant = "warrant" - // CauseDrill is a drill's cause, and the one cause a repair may not give. - CauseDrill = "drill" - // CauseOperatorAnswer is an act the operator chose on a warrant. - CauseOperatorAnswer = "operator-answer" + VerbRecord = handacts.VerbRecord + VerbDrill = handacts.VerbDrill + VerbWarrant = handacts.VerbWarrant + CauseDrill = handacts.CauseDrill + CauseOperatorAnswer = handacts.CauseOperatorAnswer ) // RepeatedWithin is how far back a cause counts as repeated (S15). const RepeatedWithin = 14 * 24 * time.Hour -// Act is one entry of the log, in the layout of the controller's (mesh-controller internal/link HandAct), so one -// reader reads both logs alike. **A copy of the layout mesh-sdk holds for an act (issue 1953, L5-S2)**, replaced by it -// when it lands. -type Act struct { - ID string `json:"id"` - At time.Time `json:"at"` - By string `json:"by"` - Verb string `json:"verb"` - Args []string `json:"arguments,omitempty"` - Why string `json:"why"` - Cause string `json:"cause"` - Condition string `json:"condition,omitempty"` - Kind string `json:"kind,omitempty"` - Carried []string `json:"carried,omitempty"` - Via string `json:"via,omitempty"` - Ask string `json:"ask,omitempty"` - Proofs []string `json:"proofs,omitempty"` - RequestedBy string `json:"requested-by,omitempty"` - Outcome string `json:"outcome,omitempty"` - // Log is which log holds it, set when it is read: this seat's, or the controller's. Never written. - Log string `json:"log,omitempty"` -} +// Act is one entry of the log: mesh-sdk's handacts.Act, the controller's HandAct unchanged. +type Act = handacts.Act // Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only // when it has none, and says ErrExists otherwise. @@ -109,9 +82,10 @@ var ErrExists = errors.New("the key was made by another writer first") // Log is mesh-hand-acts: the log and what records into it. type Log struct { Store Store - Emit func(event string, body any) error - Now func() time.Time - Logf func(format string, args ...any) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) } var seq atomic.Uint64 @@ -170,8 +144,8 @@ func (l *Log) record(act Act) error { if err := l.Store.Append(ActsLog, act.ID, act); err != nil { return fmt.Errorf("the act could not be recorded: %w", err) } - if l.Emit != nil { - if err := l.Emit(RecordedEvent, act); err != nil { + if l.Publish != nil { + if err := l.Publish(RecordedSubject, act, act.ID); err != nil { l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err) } } @@ -326,7 +300,7 @@ func (l *Log) HandActs(days int, cause, who string) Listing { all = append(all, acts...) } for _, a := range all { - if now.Sub(a.At) <= RepeatedWithin && !decision(a) { + if now.Sub(a.At) <= RepeatedWithin && a.Repair() { out.Repeated[a.Cause]++ } if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) { @@ -364,7 +338,7 @@ func (l *Log) read(log string, since time.Time) ([]Act, int, error) { return nil, 0, err } for _, raw := range entries { - keep(raw, Seat) + keep(raw, handacts.LogSeat) } return out, len(entries), nil } @@ -381,7 +355,7 @@ func (l *Log) read(log string, since time.Time) ([]Act, int, error) { return nil, len(keys), err } if found { - keep(raw, log) + keep(raw, handacts.LogController) } } return out, len(keys), nil @@ -399,29 +373,3 @@ func keyTime(key string) (time.Time, bool) { } return time.Unix(0, ns), true } - -// decided are the verbs whose acts are a person's decision by design, with the causes they hold for (every cause when -// none): the controller's handActVerbs as S15 reads them. A verb not listed is a repair. -var decided = map[string][]string{ - VerbWarrant: nil, "seat": {"seat-passed"}, "conditions silence": {CauseOperatorAnswer}, VerbRecord: {CauseDrill}, - "secret issued": nil, VerbDrill: nil, "retire approve": nil, "retire reject": nil, "cleanup delete": nil, - "dead-letters deliver": nil, "dead-letters drop": nil, "collect": nil, "mirrors": nil, "move-placement": nil, - "move-config": nil, "bus upgrade": nil, "upgrade release-backlog": nil, "secret accept": nil, - "secret rotate": {"leaked-in-logs"}, "rotate": {"leaked-in-logs"}, -} - -func decision(a Act) bool { - causes, ok := decided[a.Verb] - if !ok { - return false - } - if len(causes) == 0 { - return true - } - for _, c := range causes { - if c == a.Cause { - return true - } - } - return false -} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go index b2762666..73d42cef 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go @@ -127,8 +127,8 @@ func newLog() (*Log, *memStore, *[]Act, *time.Time) { var said []Act now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now }, - Emit: func(event string, body any) error { - if a, ok := body.(Act); ok && event == RecordedEvent { + Publish: func(subject string, body any, id string) error { + if a, ok := body.(Act); ok && subject == "mesh.seat.mesh-hand-acts.event.act-recorded" && id == a.ID { said = append(said, a) } return nil diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go index 0023c53c..b103fa23 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go @@ -76,7 +76,7 @@ func (runtimeStore) Fill(log string) (Fill, error) { func main() { l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf, - Emit: func(event string, body any) error { return stdio.Emit(event, body) }} + Publish: stdio.SeatPublish} if err := stdio.Serve("", Tools(l)); err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go index 90669933..1f6589ae 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -39,7 +39,8 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { t.Errorf("claims %+v", m.Claims) } - if !slices.Equal(m.Emits, []string{RecordedEvent}) || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { + // act-recorded is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { t.Errorf("emits %v, reads %v", m.Emits, m.Reads) } readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go index 3de340f2..64ec46d2 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -204,6 +204,13 @@ func (p *player) handle(m wire) { err = kv.Delete(p.ctx, a.Key) } p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) case "mesh/publish": var e struct { Key string `json:"key"` @@ -343,7 +350,7 @@ func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { if err != nil { t.Fatal(err) } - recorded, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + RecordedEvent) + recorded, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + RecordedEvent) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod index f79a46d2..3b27171d 100644 --- a/modules/mesh-hand-acts/go.mod +++ b/modules/mesh-hand-acts/go.mod @@ -3,7 +3,7 @@ module mesh-hand-acts go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum index ae67790f..0bfd755e 100644 --- a/modules/mesh-hand-acts/go.sum +++ b/modules/mesh-hand-acts/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json index 768f3e23..6882e201 100644 --- a/modules/mesh-hand-acts/module.json +++ b/modules/mesh-hand-acts/module.json @@ -13,9 +13,6 @@ ] } ], - "emits": [ - "act-recorded" - ], "reads": [ "mesh-controller.hand-acts", "messenger.asks" diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index e18d8cae..3c6dde6c 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -45,6 +45,8 @@ const ( ConditionsRead = "mesh-conditions.conditions" HandActsRead = "mesh-controller.hand-acts" ActedEvent = health.EventHealerActed + // ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays). + ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent // The events it hears: the keeper's, on the seat mesh-conditions. RaisedEvent = "mesh-conditions.condition-raised" ChangedEvent = "mesh-conditions.condition-changed" @@ -152,9 +154,10 @@ func Registry() []Healer { // Healers is mesh-healers: the registry, what its acts reach, and what it holds. type Healers struct { - Store Store - Ask func(key string, body any) (json.RawMessage, error) - Emit func(event string, body any) error + Store Store + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error Now func() time.Time Logf func(format string, args ...any) Registry []Healer @@ -354,10 +357,10 @@ func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attem func (h *Healers) tell(e Acted) { e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer e.Show = "mesh-conditions.conditions key=" + e.Condition - if h.Emit == nil { + if h.Publish == nil { return } - if err := h.Emit(ActedEvent, e); err != nil { + if err := h.Publish(ActedSubject, e, e.Healer+"."+e.Condition+"."+e.Outcome+"."+strconv.FormatInt(e.At.UnixNano(), 10)); err != nil { h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err) } } diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index 00612cb8..50a69f42 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -112,8 +112,8 @@ func (w *world) fresh() { w.asked = append(w.asked, key+" "+string(raw)) return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil }, - Emit: func(event string, body any) error { - if a, ok := body.(Acted); ok && event == ActedEvent { + Publish: func(subject string, body any, _ string) error { + if a, ok := body.(Acted); ok && subject == "mesh.seat.mesh-healers.event.healer-acted" { w.said = append(w.said, a) } return nil diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 2207e3c4..ddb4c7b1 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -61,9 +61,9 @@ func watched(state, key string, handle func(stdio.StateChange) error) { func main() { h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, - Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, - Emit: func(event string, body any) error { return stdio.Emit(event, body) }, - After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }} + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish, + After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }} // The lease first: until it is read the healers write would-acts only. Once it is first read, every settle is // re-armed from the attempts kept, so a restart loses no wake. var started sync.Once diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go index 86588a98..ab3528f7 100644 --- a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -47,7 +47,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { name string got, want []string }{ - {"emits", m.Emits, []string{ActedEvent}}, + {"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil}, {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent}}, {"invokes", m.Invokes, []string{AssertTool}}, {"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}}, diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go index 5c7ec972..a8e2733a 100644 --- a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -216,6 +216,13 @@ func (p *player) handle(m wire) { err = kv.Delete(p.ctx, a.Key) } p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) case "mesh/publish": var e struct { Key string `json:"key"` @@ -332,7 +339,7 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) { if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil { t.Fatal(err) } - acted, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + ActedEvent) + acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index 587a34e4..229731fa 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index ae67790f..0bfd755e 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index 186965ec..cc2438e1 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -13,9 +13,6 @@ ] } ], - "emits": [ - "healer-acted" - ], "consumes": [ "mesh-conditions.condition-raised", "mesh-conditions.condition-changed" diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go index c423e855..4225eead 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/main.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -40,8 +40,8 @@ func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(st // NewChecker is the self-check over the runtime. func NewChecker() *Checker { return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), - Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, - Emit: func(event string, body any) error { return stdio.Emit(event, body) }} + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} } func main() { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go index 10f33dc9..fb245a78 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go @@ -47,7 +47,8 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) } - if !slices.Equal(m.Emits, []string{CheckedEvent}) { + // checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 { t.Errorf("emits %v", m.Emits) } if !slices.Equal(m.Invokes, []string{StreamsTool, ClosedTool, SignalsVerb}) { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go index 8d07725d..53efb782 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -32,6 +32,9 @@ const ( VerdictsState = "verdicts" LastKey = "last" CheckedEvent = health.EventChecked + // CheckedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent ) // The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). @@ -83,9 +86,10 @@ type ( // Checker is the self-check: the registry and what its probes reach. type Checker struct { - Store Store - Ask func(key string, body any) (json.RawMessage, error) - Emit func(event string, body any) error + Store Store + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error Now func() time.Time Logf func(format string, args ...any) Host string @@ -328,10 +332,10 @@ func (c *Checker) keep(run Run) { if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil { c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err) } - if c.Emit == nil { + if c.Publish == nil { return } - if err := c.Emit(CheckedEvent, run); err != nil { + if err := c.Publish(CheckedSubject, run, run.Run); err != nil { c.mu.Lock() c.unsaid++ c.mu.Unlock() diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index 311bf924..e79bb78f 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -81,9 +81,9 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {}, Now: func() time.Time { now = now.Add(time.Second); return now }, - Emit: func(event string, body any) error { + Publish: func(subject string, body any, id string) error { raw, _ := json.Marshal(body) - said = append(said, event+" "+string(raw)) + said = append(said, subject+" "+id+" "+string(raw)) return nil }} return c, &said @@ -147,11 +147,14 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) { t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps)) } - if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], CheckedEvent+" ") { + // Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id. + if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") { t.Fatalf("said %v", *said) } + _, rest, _ := strings.Cut((*said)[0], " ") + _, rest, _ = strings.Cut(rest, " ") var body map[string]any - _ = json.Unmarshal([]byte(strings.TrimPrefix((*said)[0], CheckedEvent+" ")), &body) + _ = json.Unmarshal([]byte(rest), &body) for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} { if _, ok := body[field]; !ok { t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go index cc078779..54ae1679 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -146,6 +146,13 @@ func (p *player) handle(m wire) { err = kv.Delete(p.ctx, a.Key) } p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) case "mesh/publish": var e struct { Key string `json:"key"` @@ -230,7 +237,7 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { t.Fatal(err) } } - checked, err := nc.SubscribeSync("mesh.mod." + Module + ".event." + CheckedEvent) + checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index a4d5f850..f96f4183 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index ae67790f..0bfd755e 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba h1:uXwkcQWgbu/UthizGHFaKPiA4NBVJgqL9q2jJnnGR1o= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029204453-69fcb6c1c1ba/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json index b0e732fe..829af53e 100644 --- a/modules/mesh-self-check/module.json +++ b/modules/mesh-self-check/module.json @@ -12,9 +12,6 @@ ] } ], - "emits": [ - "checked" - ], "invokes": [ "nats.nats_streams", "nats.nats_closed_connections", From 4cbdd0fc744e8a21bdbc4dd40b91a5ea7eda478b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:16:23 +0100 Subject: [PATCH 10/20] Give each delivery's hold bound its own key's time to live and map a seat's finding to the module the placement says holds it, so the keeper holds the right findings for the right time (issue 1952 C7) --- .../cmd/mesh-conditions/hold.go | 40 +++++++++++++++++-- .../cmd/mesh-conditions/hold_test.go | 20 ++++++++++ .../cmd/mesh-conditions/keeper.go | 15 ++++--- .../cmd/mesh-conditions/keeper_test.go | 10 ++++- .../cmd/mesh-conditions/main.go | 13 +++--- .../cmd/mesh-conditions/manifest_test.go | 12 +++--- .../mesh-conditions/whole_integration_test.go | 2 +- modules/mesh-conditions/go.mod | 2 +- modules/mesh-conditions/go.sum | 4 +- modules/mesh-conditions/module.json | 4 +- 10 files changed, 96 insertions(+), 26 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go index 8cc8e7b8..8491812e 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/hold.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -52,8 +52,14 @@ func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about { out = append(out, about{Node: parts[1], Module: parts[0]}) } case health.ScopeSeat: + // The module the placement says holds the seat on that node, or the seat's own name where none is read. if len(parts) >= 2 { - out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) + node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".") + module := k.holders[node+"."+seat] + if module == "" { + module = seat + } + out = append(out, about{Node: node, Module: module}) } case health.ScopeMachine: // The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is @@ -143,9 +149,8 @@ func (k *Keeper) keepHeld(h held) error { return nil } -// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). The state `bounds` -// keeps each key that long (its ttl-seconds), since the runtime writes no key with a time to live of its own; a bound a -// manifest names needs that first. +// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state +// `bounds` is written with it as its own time to live, and its expiry is the bound passing. func (k *Keeper) bound(string) time.Duration { if k.Bound > 0 { return k.Bound @@ -211,6 +216,33 @@ func (k *Keeper) Have(key string, value []byte, deleted bool) error { return k.judge(node, module, "") } +// Placement takes one change of the placement: which module holds which seat on a node. +func (k *Keeper) Placement(key string, value []byte, deleted bool) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + return + } + k.mu.Lock() + defer k.mu.Unlock() + k.init() + for hk := range k.holders { + if strings.HasPrefix(hk, node+".") { + delete(k.holders, hk) + } + } + if deleted { + return + } + p, err := delivery.DecodePlacement(value) + if err != nil { + k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err) + return + } + for _, h := range p.Holds { + k.holders[node+"."+h.Claim] = h.Module + } +} + // BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. func (k *Keeper) BoundPassed(key string) error { node, module := splitNodeModule(key) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go index 0ee296a2..fa668544 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go @@ -71,6 +71,10 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) { if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { t.Fatal(err) } + // The delivery's bound is its own key's time to live, the module's bound (the default where it names none). + if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound { + t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"]) + } if got := r.changes(); got != "raised "+other+", raised "+machine { t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got) } @@ -106,4 +110,20 @@ func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) { t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held()) } } + // A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held + // while the assigner is delivered on ace, though the seat is named mesh-placement. + r := newRig() + placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}}, + Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}}) + if err != nil { + t.Fatal(err) + } + r.k.Placement("placement.ace", placed, false) + _ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now) + _ = r.k.Finding(selfCheck, seat, seatRaw, false) + if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" { + t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held()) + } } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 70a82f3d..802126f0 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -48,8 +48,11 @@ const ( ControllerLease = "mesh-controller.lease" ControllerLeaseKey = "holder" WantsRead = delivery.AssignerModule + ".wants" - HavesRead = "node-engine.have" - ConditionWrites = 8 + // PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the + // lead, 2026-10-29, for ADR 0564 clause 2's inputs). + PlacementRead = delivery.PlacementRead + HavesRead = "node-engine.have" + ConditionWrites = 8 ) // Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to @@ -104,8 +107,10 @@ type Keeper struct { takeover health.Takeover wants map[string]delivery.Want haves map[string]delivery.Have - stuck map[string]string - held map[string]held + // holders are, by `.`, the module the placement says holds the seat there. + holders map[string]string + stuck map[string]string + held map[string]held // passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the // line a key raised out of a hold says (hold.go). passed map[string]string @@ -121,7 +126,7 @@ func (k *Keeper) init() { k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{}, map[string]string{}, map[string]held{}, map[string]clearedMemo{} k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController} - k.passed, k.outOfHold = map[string]string{}, map[string]string{} + k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{} } if k.Now == nil { k.Now = time.Now diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 6f8dd1d8..cc90e26a 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -20,6 +20,8 @@ type fakeStore struct { vals map[string][]byte revs map[string]uint64 next uint64 + // ttls are the times to live the keys were created with. + ttls map[string]time.Duration } func newFakeStore() *fakeStore { @@ -79,7 +81,13 @@ func (f *fakeStore) Keys(state string) ([]string, error) { return out, nil } -func (f *fakeStore) CreateTTL(state, key string, value any, _ time.Duration) error { +func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + f.mu.Lock() + if f.ttls == nil { + f.ttls = map[string]time.Duration{} + } + f.ttls[state+"/"+key] = ttl + f.mu.Unlock() _, err := f.Create(state, key, value) return err } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index 2a1c654e..ac597d4b 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -44,10 +44,9 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state). func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } -// CreateTTL creates the key in a state whose own age is the time to live (`bounds`, ttl-seconds): the runtime writes no -// key with a time to live of its own. -func (runtimeStore) CreateTTL(state, key string, value any, _ time.Duration) error { - _, err := stdio.State(state).Create(key, value) +// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire. +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) return err } @@ -132,11 +131,15 @@ func Keep(k *Keeper) { }) } watch(k, BoundsState, "", func(c stdio.StateChange) error { - if c.Op != "put" && !c.Current { + if c.Op == stdio.OpExpire { return said(k.BoundPassed(c.Key)) } return nil }) + watch(k, PlacementRead, "", func(c stdio.StateChange) error { + k.Placement(c.Key, c.Value, c.Op != stdio.OpPut) + return nil + }) } func main() { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 66174fa2..94b1801c 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -29,7 +29,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { !slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) { t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves)) } - reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead} + reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead} for _, o := range Observers { reads = append(reads, o+"."+health.FindingsState) } @@ -39,12 +39,14 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) { t.Errorf("reads %v, want %v and never the healers' would", got, reads) } - ttl := map[string]int{} + ttl, keyTTL := map[string]int{}, map[string]bool{} for _, s := range m.State { - ttl[s.Name] = s.TTLSeconds + ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL } - if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 900 { - t.Errorf("history is kept %d s and a bound %d s", ttl[HistoryState], ttl[BoundsState]) + // Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age. + if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] { + t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v", + ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState]) } // Clause 12: each state with the tool that reads it, each act with the tool that starts it. readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool, diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go index 877a8ce1..f8881730 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -261,7 +261,7 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { if err != nil { t.Fatal(err) } - buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", + buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement", "mesh-healers_attempts"} for _, o := range Observers { buckets = append(buckets, o+"_findings") diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod index fcfe14e1..23a7dffb 100644 --- a/modules/mesh-conditions/go.mod +++ b/modules/mesh-conditions/go.mod @@ -3,7 +3,7 @@ module mesh-conditions go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-conditions/go.sum b/modules/mesh-conditions/go.sum index 8a5c3b7a..3ecc6a80 100644 --- a/modules/mesh-conditions/go.sum +++ b/modules/mesh-conditions/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268 h1:+jH77IJmxUiFDdbJljhynQRY0Cc5/5jtwmMV7UUqRI4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029210043-32f5ce371268/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index 20805f2b..89354c38 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -28,7 +28,8 @@ "mesh-controller.conditions", "mesh-controller.lease", "mesh-delivery-assigner.wants", - "node-engine.have" + "node-engine.have", + "mesh-delivery-assigner.placement" ], "state": [ "conditions", @@ -40,7 +41,6 @@ "held", { "name": "bounds", - "ttl-seconds": 900, "key-ttl": true }, "asked" From 371abdaad12cf41e899c65352a56bfe871a65f54 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:17:29 +0100 Subject: [PATCH 11/20] Wait on keys that expire on the bus instead of in-process timers, so the healers' settles and S15's fortnight survive a restart (issue 1952) A settle and the fortnight end are keys of wakes whose time to live is the wait, heard on expiry; attempts and would keep each key a week by its own time to live. Rearm stays only for a settle that expired while no instance watched. Pins mesh-sdk bc6c8a6. --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 +- modules/mesh-hand-acts/go.mod | 2 +- modules/mesh-hand-acts/go.sum | 4 +- modules/mesh-healers/README.md | 18 +-- .../mesh-healers/cmd/mesh-healers/healers.go | 88 ++++++++------ .../cmd/mesh-healers/healers_test.go | 107 +++++++++++++----- modules/mesh-healers/cmd/mesh-healers/main.go | 20 +++- .../cmd/mesh-healers/manifest_test.go | 17 +-- .../mesh-healers/cmd/mesh-healers/tools.go | 7 ++ .../mesh-healers/cmd/mesh-healers/wanted.go | 12 +- .../mesh-healers/whole_integration_test.go | 2 +- modules/mesh-healers/go.mod | 2 +- modules/mesh-healers/go.sum | 4 +- modules/mesh-healers/module.json | 10 +- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +- 17 files changed, 197 insertions(+), 108 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index 68e8f5fc..51d85147 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a diff --git a/checks/health/go.sum b/checks/health/go.sum index d598087e..7718be39 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod index 3b27171d..02012f64 100644 --- a/modules/mesh-hand-acts/go.mod +++ b/modules/mesh-hand-acts/go.mod @@ -3,7 +3,7 @@ module mesh-hand-acts go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum index 0bfd755e..3ecc6a80 100644 --- a/modules/mesh-hand-acts/go.sum +++ b/modules/mesh-hand-acts/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md index b266558b..30aac2c8 100644 --- a/modules/mesh-healers/README.md +++ b/modules/mesh-healers/README.md @@ -18,21 +18,23 @@ mid-act has spent it and a second instance cannot begin it again. The condition **Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. -Both buckets age as a whole, each key a week after its last write: the runtime gives a bundle no time to live per -key, so the bucket's age is the deleter for this release. The end of a settle and the end of S15's fortnight wake the -healers through an in-process timer, armed by the act; at start, once the lease is first read, every settle still -running is re-armed from `attempts` and one that ended while they were down is judged at once, and the hand-act log's -watch hands over every entry again, which re-arms S15. So a restart loses no wake. +Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared +`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits +are keys of `wakes` whose time to live is the wait: `settle.` ends with a settle and judges the condition +again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears +each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while +no instance watched is judged at the start, once the lease is first read. S15: it watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a fortnight; a drill and a person's decision by design never count. ## What it keeps -- `mesh-healers_attempts`: every act and escalation, kept a week (the bucket's age). +- `mesh-healers_attempts`: every act and escalation, each key kept a week. - `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease, - kept a week (the bucket's age); nothing else deletes it, and nothing is written there once the lease is handed away. + each key kept a week. - `mesh-healers_findings`: the brake, its own blindness, and S15. +- `mesh-healers_wakes`: the waits, each key living as long as it waits. ## Its verbs and tools @@ -40,4 +42,4 @@ fortnight; a drill and a person's decision by design never count. `recount=true` counts the hand-act log again. - `mesh-healers.heal`: one act now on an open condition, by key and why. - `mesh-healers.drill`: refused until the hand-act log's home is delivered (issue 1953), naming `mesh-controller.drill`. -- `healers_findings`, `healers_would`: the reads of its buckets. +- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index 3c6dde6c..d871de01 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -25,6 +25,7 @@ import ( "sync" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -68,16 +69,32 @@ const ( OutcomeEscalated = health.OutcomeEscalated ) -// Store is the bus's buckets as this module reaches them, through its runtime. Create writes a key only when it has -// none, and says ErrExists otherwise. +// Store is the bus's buckets as this module reaches them, through its runtime. CreateTTL writes a key only when it has +// none, and says ErrExists otherwise; it and PutTTL give the key its own time to live, at whose end the bus removes it +// and a watch of the bucket hears it expire (a bucket declared key-ttl). type Store interface { Get(state, key string) ([]byte, bool, error) Put(state, key string, value any) error - Create(state, key string, value any) error + CreateTTL(state, key string, value any, ttl time.Duration) error + PutTTL(state, key string, value any, ttl time.Duration) error Delete(state, key string) error Keys(state string) ([]string, error) } +// KeptFor is how long an attempt and a would-act are kept: each key a week from its last write (ADR 0564 Limits). +const KeptFor = 7 * 24 * time.Hour + +// WakesState holds the healers' waits as keys whose time to live is the wait: `settle.` ends with a +// settle, `fortnight` when S15's oldest counted act leaves the fortnight. Its expiry, heard on the bucket's watch, is +// the wake: no timer runs in the process, and a restart loses nothing a key on the bus holds. +const WakesState = "wakes" + +// The wakes' keys. +const ( + settlePrefix = "settle." + fortnightKey = "fortnight" +) + // ErrExists is a create of a key another writer made first. var ErrExists = errors.New("the key was made by another writer first") @@ -161,8 +178,6 @@ type Healers struct { Now func() time.Time Logf func(format string, args ...any) Registry []Healer - // After wakes the healers when a settle ends, to judge the condition again; nil wakes nothing. - After func(d time.Duration, f func()) mu sync.Mutex // lease is the controller's lease as last read: acting is on only while it is handed away. @@ -172,9 +187,6 @@ type Healers struct { declined map[string]string // acts are the hand-act log as watched, for S15. acts map[string]HandAct - // woken is the settle ends already waited for, by budget key. - woken map[string]time.Time - s15At time.Time } // LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564 @@ -253,7 +265,7 @@ func (h *Healers) Judge(c Condition, by string) (string, error) { now := h.Now() if !h.acting { w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()} - if err := h.Store.Put(WouldState, row.ID+"."+c.Key, w); err != nil { + if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil { return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err) } return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil @@ -305,7 +317,7 @@ func (h *Healers) act(row Healer, c Condition, budget string, spent int, by stri if by != "healer "+row.ID { a.Said = "asked by " + by } - if err := h.Store.Create(AttemptsState, key, a); err != nil { + if err := h.Store.CreateTTL(AttemptsState, key, a, KeptFor); err != nil { if errors.Is(err, ErrExists) { return "another instance began this act first", nil } @@ -324,7 +336,7 @@ func (h *Healers) act(row Healer, c Condition, budget string, spent int, by stri said = a.Said + "; " + said } a.Said, a.Finished = said+" ("+window+")", &finished - if err := h.Store.Put(AttemptsState, key, a); err != nil { + if err := h.Store.PutTTL(AttemptsState, key, a, KeptFor); err != nil { h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err) } h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window}) @@ -345,7 +357,7 @@ func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attem window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window) a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator", Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now} - if err := h.Store.Create(AttemptsState, attemptKey(row.ID, c.Key, now), a); err != nil && !errors.Is(err, ErrExists) { + if err := h.Store.CreateTTL(AttemptsState, attemptKey(row.ID, c.Key, now), a, KeptFor); err != nil && !errors.Is(err, ErrExists) { return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err) } h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated, @@ -365,13 +377,13 @@ func (h *Healers) tell(e Acted) { } } -// Rearm is the healers' start, once the lease is first read: every settle still running is re-armed from the attempts -// kept, and one whose end passed while the healers were down is judged at once, so a restart loses no wake. S15's -// fortnight end is re-armed by the hand-act log's watch, which hands over every current entry when it starts. +// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has +// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still +// running keeps its key on the bus and needs nothing. func (h *Healers) Rearm() { all, err := h.attempts() if err != nil { - h.Logf("[%s] what the healers did cannot be read, so no settle is re-armed: %v", Module, err) + h.Logf("[%s] what the healers did cannot be read, so no ended settle is judged: %v", Module, err) return } now := h.Now() @@ -382,43 +394,45 @@ func (h *Healers) Rearm() { } } var due []string - h.mu.Lock() for _, a := range newest { - var row Healer for _, r := range h.Registry { - if r.ID == a.Healer && r.repair != nil { - row = r + if r.ID == a.Healer && r.repair != nil && !now.Before(a.Begun.Add(r.Settle)) && now.Sub(a.Begun) <= r.Window { + due = append(due, a.Condition) } } - if row.ID == "" { - continue - } - if end := a.Begun.Add(row.Settle); end.After(now) { - h.wake(a.BudgetKey, end, a.Condition) - } else if now.Sub(a.Begun) <= row.Window { - due = append(due, a.Condition) - } } - h.mu.Unlock() sort.Strings(due) for _, key := range due { h.again(key) } } -// wake judges a condition again when its settle ends: the one wait of the healers, set by an act, never a look on a -// schedule. The settle is read from the attempt's time, and Rearm sets it again after a restart. -func (h *Healers) wake(budget string, at time.Time, key string) { - if h.After == nil || !h.woken[budget].Before(at) { +// wake writes the wait for a settle's end as a key whose time to live is the wait; its expiry judges the condition +// again (WakeHeard). Written again with the same end, it is the same wait. +func (h *Healers) wake(_ string, at time.Time, key string) { + ttl := at.Sub(h.Now()).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, settlePrefix+key, map[string]any{"condition": key, "at": at.UTC()}, ttl); err != nil { + h.Logf("[%s] the settle of %s is not written as a wait, so its end is judged at the next event: %v", Module, key, err) + } +} + +// WakeHeard is a change of the wakes bucket: an expiry is a wait ended — a settle's, whose condition is judged again, +// or S15's fortnight, counted again. +func (h *Healers) WakeHeard(key, op string) { + if op != OpExpire { return } - if h.woken == nil { - h.woken = map[string]time.Time{} + switch { + case key == fortnightKey: + h.Wanted() + case strings.HasPrefix(key, settlePrefix): + h.again(strings.TrimPrefix(key, settlePrefix)) } - h.woken[budget] = at - h.After(at.Sub(h.Now())+time.Second, func() { h.again(key) }) } +// OpExpire is the watch's word for a key whose own time to live ended. +const OpExpire = stdio.OpExpire + // again judges a condition again from the keeper's bucket, if it is still open. func (h *Healers) again(key string) { raw, found, err := h.Store.Get(ConditionsRead, key) diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index 50a69f42..16ea85d3 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -17,6 +17,7 @@ type memStore struct { mu sync.Mutex kv map[string]map[string][]byte keysErr map[string]error + ttl map[string]time.Duration } func newStore() *memStore { @@ -50,6 +51,39 @@ func (s *memStore) put(state, key string, value any, create bool) error { func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } +// CreateTTL and PutTTL keep the key's time to live beside it, for a test to read. +func (s *memStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, true); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) PutTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, false); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) setTTL(state, key string, ttl time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ttl == nil { + s.ttl = map[string]time.Duration{} + } + s.ttl[state+"/"+key] = ttl +} + +// ttlOf is a key's time to live as last written; zero for none. +func (s *memStore) ttlOf(state, key string) time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return s.ttl[state+"/"+key] +} + func (s *memStore) Delete(state, key string) error { s.mu.Lock() defer s.mu.Unlock() @@ -93,7 +127,6 @@ type world struct { now time.Time asked []string said []Acted - wakes []time.Duration } func newWorld() *world { @@ -104,7 +137,6 @@ func newWorld() *world { // fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten. func (w *world) fresh() { - w.wakes = nil w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, Now: func() time.Time { return w.now }, Ask: func(key string, body any) (json.RawMessage, error) { @@ -118,7 +150,7 @@ func (w *world) fresh() { } return nil }, - After: func(d time.Duration, _ func()) { w.wakes = append(w.wakes, d) }} + } } func (w *world) handAway() { @@ -189,44 +221,46 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { w.handAway() c := lost("records") _ = w.store.Put(ConditionsRead, c.Key, c) + settle := settlePrefix + c.Key steps := []struct { name string after time.Duration cond Condition attempts int outcome string // the newest said outcome, empty for none new - // restart starts the healers again over the same buckets instead of an event, and wake is the settle end - // they must re-arm (none when zero). - restart bool - wake time.Duration + // how is what happens: an event, a restart over the same buckets, or the settle key's expiry heard. + how string + // wait is the settle key's time to live after the step, zero for no check. + wait time.Duration }{ - {"acts", 0, c, 1, OutcomeActed, false, 0}, - {"within its settle: waits", 2 * time.Minute, c, 1, "", false, 0}, - {"a restart within its settle re-arms its end", time.Minute, c, 1, "", true, 3*time.Minute + time.Second}, - {"a restart past its settle judges it at once: escalates", 4 * time.Minute, c, 2, OutcomeEscalated, true, 0}, + {"acts: its settle is a key that expires", 0, c, 1, OutcomeActed, "event", 6*time.Minute + time.Second}, + {"within its settle: waits for the same end", 2 * time.Minute, c, 1, "", "event", 4*time.Minute + time.Second}, + {"a restart within its settle: the key on the bus still waits", time.Minute, c, 1, "", "restart", 4*time.Minute + time.Second}, + {"its key expires: judged again, budget spent, escalates", 4 * time.Minute, c, 2, OutcomeEscalated, "expire", 0}, {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", - Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", false, 0}, + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", "event", 0}, } - var first []time.Duration for _, s := range steps { w.now = w.now.Add(s.after) before := len(w.said) - if s.restart { - if len(first) == 0 { - first = w.wakes - } + switch s.how { + case "restart": w.fresh() w.handAway() w.h.Rearm() - if s.wake != 0 && !slices.Equal(w.wakes, []time.Duration{s.wake}) { - t.Errorf("%s: wakes %v, want %v", s.name, w.wakes, s.wake) + case "expire": + w.h.WakeHeard(settle, OpExpire) + default: + if _, err := w.h.Judge(s.cond, ""); err != nil { + t.Fatalf("%s: %v", s.name, err) } - } else if _, err := w.h.Judge(s.cond, ""); err != nil { - t.Fatalf("%s: %v", s.name, err) } if n := len(w.store.keys(AttemptsState)); n != s.attempts { t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts) } + if s.wait != 0 && w.store.ttlOf(WakesState, settle) != s.wait { + t.Errorf("%s: the settle waits %s, want %s", s.name, w.store.ttlOf(WakesState, settle), s.wait) + } switch { case s.outcome == "" && len(w.said) != before: t.Errorf("%s: said %+v", s.name, w.said[before:]) @@ -234,13 +268,30 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) } } - if len(first) == 0 || first[0] != 6*time.Minute+time.Second { - t.Errorf("the settle's end wakes %v, want once after 6m1s", first) + // Each attempt is kept a week from its last write. + for _, k := range w.store.keys(AttemptsState) { + if ttl := w.store.ttlOf(AttemptsState, k); ttl != KeptFor { + t.Errorf("the attempt %s is kept %s, want a week", k, ttl) + } + } + // A settle whose key expired while no instance watched is judged at the start. + r := newWorld() + r.handAway() + _ = r.store.Put(ConditionsRead, c.Key, c) + if _, err := r.h.Judge(c, ""); err != nil { + t.Fatal(err) + } + r.now = r.now.Add(7 * time.Minute) + r.fresh() + r.handAway() + r.h.Rearm() + if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated { + t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said) } // A second instance cannot begin the act an attempt's key already holds. x := newWorld() x.handAway() - _ = x.store.Create(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}) + _ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor) if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 { t.Errorf("a second instance acted: %q, asked %v", said, x.asked) } @@ -249,8 +300,8 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { b.handAway() for i := 0; i < BrakeLimit; i++ { b.now = b.now.Add(time.Minute) - _ = b.store.Create(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x", - Outcome: OutcomeActed, Begun: b.now}) + _ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x", + Outcome: OutcomeActed, Begun: b.now}, KeptFor) } if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 { t.Errorf("past the brake: %q, asked %v", said, b.asked) @@ -343,8 +394,8 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { if !strings.Contains(string(raw), "healer H4 answers this cause") { t.Errorf("the finding does not say its healer is not enough: %s", raw) } - if n := len(w.wakes); n == 0 || w.wakes[n-1] != 4*24*time.Hour+time.Second { - t.Errorf("wakes %v, want the oldest act's leaving the fortnight in 4 days", w.wakes) + if ttl := w.store.ttlOf(WakesState, fortnightKey); ttl != 4*24*time.Hour+time.Second { + t.Errorf("the fortnight waits %s, want the oldest act's leaving it in 4 days", ttl) } w.now = w.now.Add(4*24*time.Hour + time.Second) w.h.Wanted() diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index ddb4c7b1..7165f20b 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -35,14 +35,19 @@ func (runtimeStore) Put(state, key string, value any) error { return err } -func (runtimeStore) Create(state, key string, value any) error { - _, err := stdio.State(state).Create(key, value) +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) if errors.Is(err, stdio.ErrChanged) { return ErrExists } return err } +func (runtimeStore) PutTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).PutTTL(key, value, ttl) + return err +} + func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } @@ -62,16 +67,19 @@ func watched(state, key string, handle func(stdio.StateChange) error) { func main() { h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, - Publish: stdio.SeatPublish, - After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }} - // The lease first: until it is read the healers write would-acts only. Once it is first read, every settle is - // re-armed from the attempts kept, so a restart loses no wake. + Publish: stdio.SeatPublish} + // The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended + // while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry. var started sync.Once go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { h.LeaseHeard(c.Value, c.Op == "put", nil) started.Do(func() { go h.Rearm() }) return nil }) + go watched(WakesState, "", func(c stdio.StateChange) error { + h.WakeHeard(c.Key, c.Op) + return nil + }) go watched(HandActsRead, "", func(c stdio.StateChange) error { h.HandActHeard(c.Key, c.Op, c.Value) return nil diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go index ab3528f7..08d16430 100644 --- a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -56,23 +56,24 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { t.Errorf("%s %v, want %v", c.name, c.got, c.want) } } - states := map[string]int{} + states := map[string]bool{} for _, s := range m.State { var d struct { - Name string `json:"name"` - TTL int `json:"ttl-seconds"` + Name string `json:"name"` + KeyTTL bool `json:"key-ttl"` } if json.Unmarshal(s, &d.Name) != nil { _ = json.Unmarshal(s, &d) } - states[d.Name] = d.TTL + states[d.Name] = d.KeyTTL } - week := 7 * 24 * 60 * 60 - if states[AttemptsState] != week || states[WouldState] != week { - t.Errorf("attempts kept %d s and would %d s, want a week each: would has no other deleter", states[AttemptsState], - states[WouldState]) + // Each key of attempts and would lives a week from its last write (KeptFor, would has no other deleter), and each + // wait in wakes as long as it waits: their buckets take a key's own time to live. + if !states[AttemptsState] || !states[WouldState] || !states[WakesState] || states[FindingsState] { + t.Errorf("the states take a key's own time to live as %v, want attempts, would and wakes alone", states) } readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool, + WakesState: WakesTool, "a settle's end": HealTool, "an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool} tools := map[string]bool{} for _, tool := range Tools(&Healers{}) { diff --git a/modules/mesh-healers/cmd/mesh-healers/tools.go b/modules/mesh-healers/cmd/mesh-healers/tools.go index 949f4e2a..3dd70607 100644 --- a/modules/mesh-healers/cmd/mesh-healers/tools.go +++ b/modules/mesh-healers/cmd/mesh-healers/tools.go @@ -25,6 +25,7 @@ const ( DrillTool = Seat + ".drill" FindingsTool = "healers_findings" WouldTool = "healers_would" + WakesTool = "healers_wakes" ) func obj(props map[string]any) map[string]any { @@ -66,6 +67,12 @@ func Tools(h *Healers) []stdio.Tool { "each cause repaired by hand twice in a fortnight (S15); or one whole by key.", Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }}, + {Name: WakesTool, + Description: "What the healers wait for (their bucket wakes): each settle still running (`settle.`) " + + "and S15's fortnight end (`fortnight`), each a key whose time to live is the wait and whose expiry wakes them; " + + "or one whole by key. heal acts on a condition now, and healers with recount=true counts S15 now.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }}, {Name: WouldTool, Description: "What the healers would have done while the controller holds its lease (their bucket would, each " + "key kept a week): one per healer and condition, its newest; or one whole by key.", diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go index d07ec541..0c91821e 100644 --- a/modules/mesh-healers/cmd/mesh-healers/wanted.go +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -5,7 +5,7 @@ package main // watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the // finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design // (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the -// fortnight: the healers wake once, when the oldest counted act leaves it. From the log's home (issue 1953) they +// fortnight: the key `fortnight` of the healers' wakes expires when the oldest counted act leaves it. From the log's home (issue 1953) they // watch that instead. import ( @@ -85,7 +85,7 @@ func (h *Healers) HandActHeard(key, op string, value []byte) { } // Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, -// deleting the rest; it wakes again when the oldest counted act of a wanted cause leaves the fortnight. +// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. func (h *Healers) Wanted() []Observation { h.mu.Lock() defer h.mu.Unlock() @@ -137,9 +137,11 @@ func (h *Healers) Wanted() []Observation { } } h.see("S15", found) - if h.After != nil && !next.IsZero() && !next.Equal(h.s15At) { - h.s15At = next - h.After(next.Sub(now)+time.Second, func() { h.Wanted() }) + if !next.IsZero() { + ttl := next.Sub(now).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil { + h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err) + } } return found } diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go index a8e2733a..db056254 100644 --- a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -318,7 +318,7 @@ func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) { if err != nil { t.Fatal(err) } - for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, + for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState, "mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { t.Fatal(err) diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index 229731fa..490e0d52 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index 0bfd755e..3ecc6a80 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index cc2438e1..4319f46f 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -28,13 +28,17 @@ "state": [ { "name": "attempts", - "ttl-seconds": 604800 + "key-ttl": true }, { "name": "would", - "ttl-seconds": 604800 + "key-ttl": true }, - "findings" + "findings", + { + "name": "wakes", + "key-ttl": true + } ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index f96f4183..c6c30651 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 0bfd755e..3ecc6a80 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c h1:Yd6AslGN7pCniQ9Q7ch05xAWHP57grXRoH3AsDQrSeY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211044-fbd00c89436c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 0842361f8564fcd6a405d0a40c4641fc96a104d5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:18:59 +0100 Subject: [PATCH 12/20] =?UTF-8?q?Keep=20the=20hand-act=20log's=20acts=2040?= =?UTF-8?q?0=20days,=20as=20ADR=200565=20=C2=A710=20has=20it,=20now=20that?= =?UTF-8?q?=20a=20module=20log=20may=20say=20its=20age;=20pin=20mesh-sdk?= =?UTF-8?q?=202e5cc4f=20(issue=201953)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 ++-- modules/mesh-hand-acts/README.md | 4 ++-- .../cmd/mesh-hand-acts/manifest_test.go | 17 ++++++++++++----- modules/mesh-hand-acts/go.mod | 2 +- modules/mesh-hand-acts/go.sum | 4 ++-- modules/mesh-hand-acts/module.json | 5 ++++- modules/mesh-healers/go.mod | 2 +- modules/mesh-healers/go.sum | 4 ++-- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 ++-- 11 files changed, 30 insertions(+), 20 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index 51d85147..f9aedc80 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 diff --git a/checks/health/go.sum b/checks/health/go.sum index 7718be39..3725c41e 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-hand-acts/README.md b/modules/mesh-hand-acts/README.md index d01b8b49..b435e08d 100644 --- a/modules/mesh-hand-acts/README.md +++ b/modules/mesh-hand-acts/README.md @@ -8,8 +8,8 @@ alone writes the log. - the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream): every act appended in its order under its key `act--` (`warrant--` for a warrant), in the layout - of the controller's log. A log ages nothing and refuses a new act when full, so every act is held 400 days and longer; - `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`. + of the controller's log. The log keeps each act 400 days (`max-age-days`) and refuses a new act when + full rather than drop one; `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`. - `mesh-hand-acts_warrants`: one key `.` per warrant, made once by create before its act is appended, so of several instances one records it; a warrant whose act was lost is written again from the router's record. diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go index 1f6589ae..cd19669a 100644 --- a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -25,9 +25,12 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { Scope string `json:"scope"` Serves []string `json:"serves"` } `json:"claims"` - Emits []string `json:"emits"` - Reads []string `json:"reads"` - Logs []string `json:"logs"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + Logs []struct { + Name string `json:"name"` + MaxAgeDays int `json:"max-age-days"` + } `json:"logs"` State []json.RawMessage `json:"state"` } if err := json.Unmarshal(raw, &m); err != nil { @@ -49,8 +52,9 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { for _, tool := range Tools(&Log{}) { tools[tool.Name] = true } - if !slices.Equal(m.Logs, []string{ActsLog}) { - t.Errorf("logs %v, want the acts as a log", m.Logs) + // The acts are a log kept 400 days (ADR 0565 §10). + if len(m.Logs) != 1 || m.Logs[0].Name != ActsLog || m.Logs[0].MaxAgeDays != 400 { + t.Errorf("logs %+v, want the acts as a log kept 400 days", m.Logs) } for _, raw := range m.State { var name string @@ -61,6 +65,9 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { t.Errorf("the state %s has no tool that reads it", name) } } + if !tools[readBy[ActsLog]] { + t.Errorf("the log %s has no tool that reads it", ActsLog) + } for _, r := range m.Reads { if !tools[readBy[r]] { t.Errorf("the read %s has no tool that answers it", r) diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod index 02012f64..ae12ac5a 100644 --- a/modules/mesh-hand-acts/go.mod +++ b/modules/mesh-hand-acts/go.mod @@ -3,7 +3,7 @@ module mesh-hand-acts go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum index 3ecc6a80..f53fc84a 100644 --- a/modules/mesh-hand-acts/go.sum +++ b/modules/mesh-hand-acts/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json index 6882e201..96aff8b1 100644 --- a/modules/mesh-hand-acts/module.json +++ b/modules/mesh-hand-acts/module.json @@ -18,7 +18,10 @@ "messenger.asks" ], "logs": [ - "acts" + { + "name": "acts", + "max-age-days": 400 + } ], "state": [ "warrants" diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index 490e0d52..84e2b59c 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index 3ecc6a80..f53fc84a 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index c6c30651..4f9316ba 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 3ecc6a80..f53fc84a 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a h1:haFs0zoHDUOrdtKn0Cw9lhCg/K3Me6Ff3Mg+IdVlK34= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211442-bc6c8a6d244a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From d34645361dd2fa136b4547631565b76388c2b0ce Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:27:23 +0100 Subject: [PATCH 13/20] Build config's switch from the controller's records: declared, compare, take-over and hand-back, so the operator approves the last move only once nothing differs (issue 1953) compare also resolves every placed module's values on every node both ways, as ADR 0565's insight of 2026-10-29 asks after #558. After the take-over the predefined list and pull requests' config come from the indexer and pull-declares is refused; a hand-back returns to the mirrors until the lease write. From the lease write a trusted value is set only on an Approve. Acts are recorded through mesh-hand-acts. Pins mesh-sdk 2e5cc4f. --- modules/mesh-config/README.md | 18 + modules/mesh-config/cmd/mesh-config/config.go | 11 +- modules/mesh-config/cmd/mesh-config/main.go | 29 +- .../cmd/mesh-config/manifest_test.go | 61 ++ .../mesh-config/cmd/mesh-config/proposals.go | 1 + modules/mesh-config/cmd/mesh-config/pulls.go | 12 + .../mesh-config/cmd/mesh-config/reads-outside | 2 + .../mesh-config/cmd/mesh-config/sources.go | 58 +- .../cmd/mesh-config/switch_tools.go | 62 ++ .../mesh-config/cmd/mesh-config/takeover.go | 771 ++++++++++++++++++ .../cmd/mesh-config/takeover_test.go | 238 ++++++ modules/mesh-config/go.mod | 11 +- modules/mesh-config/go.sum | 28 +- modules/mesh-config/module.json | 19 +- 14 files changed, 1293 insertions(+), 28 deletions(-) create mode 100644 modules/mesh-config/cmd/mesh-config/manifest_test.go create mode 100644 modules/mesh-config/cmd/mesh-config/reads-outside create mode 100644 modules/mesh-config/cmd/mesh-config/switch_tools.go create mode 100644 modules/mesh-config/cmd/mesh-config/takeover.go create mode 100644 modules/mesh-config/cmd/mesh-config/takeover_test.go diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md index 42bd1ed0..72220615 100644 --- a/modules/mesh-config/README.md +++ b/modules/mesh-config/README.md @@ -61,3 +61,21 @@ them, newest first; with `proposal`, one read whole, with whether the router's a Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## The switch from the controller's records (novox/hq ADR 0565 §3, §4, §13) + +- `mesh-config_declared`: each placed module's declarations at the trunk, with the manifest and commit they come + from, written only when they differ. +- `mesh-config_takeover`: the key `switch` names the source that holds: none (the interim since mesh-catalog #558: + declarations from the modules' own manifests, the list of predefined keys and pull requests' config from the + controller), `indexer` after the operator's take-over (the list and pull requests' config from the indexer's + `predefined` and `config-candidates`; the controller's `pull-declares` refused), `controller` after a hand-back + (declarations, list and pull requests' config from the controller's records). Each take-over ask is kept beside it. +- `config_compare` lists every difference between the own sources and the controller's records: each module's + declarations, the list, each open pull request's config, and every placed module's value on every node resolved + both ways. `config_take_over` asks the operator's Approve, refused while any difference stands and switched on the + Approve only if none stands then. `config_hand_back` returns to the controller's records until its lease is handed + away. `config_declared` and `config_resync` read and write again what it keeps. +- From the controller's lease write a trusted value is set only through propose and the operator's Approve. +- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the + controller's while that seat has no holder and the controller holds its lease. diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index 0f2435cc..1928a3df 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level, strings.Join(problems, "; ")) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+ "the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+ "propose it instead, with config's action propose", call.Key, call.Caller) @@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { if !had { return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+ "operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+ "config's action propose and clear", call.Key, call.Caller) @@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return answer, nil } +// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is +// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever +// calls (novox/hq ADR 0565 §13). +func (c *Config) atTheTerminal(call Call) bool { + return call.Caller == ControllerCaller && !c.leaseAway() +} + // declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared // keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10). func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) { diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 036acd86..c0bba92e 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -19,6 +19,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. @@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) { func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) } -// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb. +// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat +// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11). +var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool { + e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +}} + +// HandAct records a change in the hand-act log (to-be 45 §7). func (runtimeStore) HandAct(what, why, cause string) error { - _, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause}) + _, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause}) return err } @@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error { func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) } func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() } func (runtimeAsks) Warranted(ask string) error { - _, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask}) + _, err := recorder.RecordWarrant(Module, ask) return err } @@ -118,7 +132,8 @@ func resolving(c *Config) { } // A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is // every change of what a module declares where (issue 1953); the builds themselves are read, not watched. - for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} { + for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead, + CandidatesRead, MirrorDeclaredRead, TakeoverState} { go func(read string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { time.Sleep(wait) @@ -221,9 +236,9 @@ func hearing(c *Config) { } } -// Tools is the seat's verb as the runtime serves it. +// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them. func Tools(c *Config) []stdio.Tool { - return []stdio.Tool{{ + return append(switchTools(c), stdio.Tool{ Name: Seat + "." + Verb, Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + @@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool { call.Caller = stdio.CallerOf(args) return c.Do(call) }, - }} + }) } // callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go new file mode 100644 index 00000000..23f06ef3 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself +// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools. +func TestEveryStateAndActHasItsTool(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + State []string `json:"state"` + Logs []string `json:"logs"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + via := map[string]string{ + ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list", + ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history", + DeclaredState: DeclaredTool, TakeoverState: DeclaredTool, + PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool, + ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool, + IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool, + "a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool, + } + served := map[string]bool{} + for _, tool := range Tools(&Config{}) { + served[tool.Name] = true + } + for _, name := range m.Tools { + if !served[name] { + t.Errorf("the manifest lists the tool %s, which is not served", name) + } + } + for _, list := range [][]string{m.State, m.Logs, m.Reads} { + for _, s := range list { + if _, ok := via[s]; !ok { + t.Errorf("%s has no tool that reads it", s) + } + } + } + for what, tool := range via { + name, _, _ := strings.Cut(tool, " ") + if name == "config" { + name = Seat + "." + Verb + } + if !served[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index d6cb304d..18619024 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) { for _, r := range ready { said = append(said, c.decide(r.p, r.w)) } + said = append(said, c.decideTakeovers()...) c.recordWarrants(now) return said, nil } diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go index f803b5de..df868157 100644 --- a/modules/mesh-config/cmd/mesh-config/pulls.go +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -151,6 +151,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+ "mesh-config what the head declares; %s is not the controller", call.Caller) } + if sw, err := c.source(); err != nil { + return nil, err + } else if sw.Source == SourceIndexer { + return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " + + "declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)") + } if _, _, _, err := parsePull(call.Pull); err != nil { return nil, err } @@ -185,6 +191,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { pull := strings.TrimSpace(call.Pull) c.mu.Lock() defer c.mu.Unlock() + return c.declarePull(pull, head, declares) +} + +// declarePull keeps what a pull request's head declares and judges again the values held for it; from the +// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock. +func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) { if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares, At: c.now()}); err != nil { return nil, fmt.Errorf("the declarations were not kept: %v", err) diff --git a/modules/mesh-config/cmd/mesh-config/reads-outside b/modules/mesh-config/cmd/mesh-config/reads-outside new file mode 100644 index 00000000..ca488b77 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-config/module.json diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index 761706d8..e32b4dda 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -12,10 +12,19 @@ import ( // Reading the sources, writing the resolved values, and publishing the list of predefined keys. -// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the -// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the -// controller (issue 1953, ADR 0565 §3). +// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names +// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations +// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror. func (c *Config) sources() (Sources, error) { + sw, err := c.source() + if err != nil { + return Sources{}, err + } + return c.sourcesFrom(sw.Source) +} + +// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror. +func (c *Config) sourcesFrom(source string) (Sources, error) { s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{}, Predefined: map[string]Published{}, Values: map[string]Value{}} var problems []string @@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) { if err := c.readDeclarations(&s); err != nil { return s, fmt.Errorf("the modules' declarations cannot be read: %w", err) } + if source == SourceController { + c.mirrorDeclarations(&s) + } for _, u := range s.Unresolved { problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why)) } @@ -350,6 +362,10 @@ type Resolution struct { Moved []Moved `json:"moved"` Published string `json:"published,omitempty"` Withheld string `json:"withheld,omitempty"` + // Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates + // did after the take-over. + Declared []string `json:"declared,omitempty"` + Candidates []string `json:"candidates,omitempty"` } // Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources @@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) { c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " + "keys changed", At: c.now(), Moved: moved}) } + if written, derr := c.writeDeclared(s); derr != nil { + c.logf("[mesh-config] declared is not all written: %v", derr) + } else if len(written) > 0 { + r.Declared = written + } + if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer { + r.Candidates = c.takeCandidates() + } return r, err } -// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published -// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the +// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the +// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins; +// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the // list published stays, and why is said in every listing and in the log, because a value for a key no longer on // the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused). // It answers what it published, empty when nothing changed. func (c *Config) publish() (string, error) { - e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + sw, err := c.source() if err != nil { - return "", fmt.Errorf("the controller's list cannot be read: %w", err) + return "", err } - if e == nil { + read, whose := c.mirrorList, "the controller's list" + if sw.Source == SourceIndexer { + read, whose = c.indexerList, "the indexer's list" + } + list, found, err := read() + if err != nil { + return "", fmt.Errorf("%s cannot be read: %w", whose, err) + } + if !found { return "", nil // no list kept yet } - var list PredefinedList - if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { - return "", fmt.Errorf("the controller's list is not one: %s", e.Value) - } current := map[string]Published{} if err := c.each(PredefinedState, func(key string, raw []byte) error { var p Published @@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) { }); err != nil { return "", err } - generation := 0 + generation, sameCommit := 0, len(current) > 0 for _, p := range current { generation = max(generation, p.Generation) + sameCommit = sameCommit && p.Commit == list.Commit } - if len(current) > 0 && generation >= list.Generation { + if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) { c.withheld = "" return "", nil } diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go new file mode 100644 index 00000000..80983e4f --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -0,0 +1,62 @@ +package main + +// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, the read of `declared` and +// resync. Module tools until mesh-sdk's seat mesh-config names them as its verbs. + +import ( + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The switch's tools. +const ( + CompareTool = "config_compare" + TakeOverTool = "config_take_over" + HandBackTool = "config_hand_back" + DeclaredTool = "config_declared" + ResyncTool = "config_resync" +) + +func switchTools(c *Config) []stdio.Tool { + why := map[string]any{"type": "object", "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}} + whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: CompareTool, + Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " + + "each placed module's declarations (declared against the mirror declared-config), the list of predefined " + + "keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " + + "against the controller's pull-declares), and every placed module's value on every node resolved both " + + "ways; each with its reason. The take-over is refused while any stands.", + Run: func(map[string]any) (any, error) { return c.Compare() }}, + {Name: TakeOverTool, + Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " + + "the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " + + "Refused while compare shows a difference; on Approve switched if nothing differs then.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: HandBackTool, + Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " + + "pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: DeclaredTool, + Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " + + "commit they come from and how many keys, or one module whole; and which source holds now.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module, whole"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + return c.Declared(strings.TrimSpace(m)) + }}, + {Name: ResyncTool, + Description: "Read every source again and write what differs: the published list of predefined keys, the " + + "resolved values, declared, and after the take-over the indexer's config candidates.", + Run: func(map[string]any) (any, error) { return c.Resync() }}, + } +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover.go b/modules/mesh-config/cmd/mesh-config/takeover.go new file mode 100644 index 00000000..dad88461 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover.go @@ -0,0 +1,771 @@ +package main + +// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of +// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every +// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list, +// each open pull request's config declarations, and every placed module's resolved value on every node both ways; +// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the +// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write. +// +// **Which source holds** is the key `switch` alone, read by mesh-config alone: +// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list +// of predefined keys from the controller's mirror, a pull request's declarations from the controller's +// pull-declares; +// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from +// the indexer's `config-candidates`; pull-declares refused; +// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror, +// pull-declares taken. + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The states and reads of the switch. +const ( + // DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3). + DeclaredState = "declared" + // TakeoverState holds the key SwitchKey and each take-over ask, `ask.`. + TakeoverState = "takeover" + SwitchKey = "switch" + + // MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module: + // what `compare` holds the own source to, and the source after a hand-back. + MirrorDeclaredRead = "mesh-controller.declared-config" + // IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2). + IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState + CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState + // LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13). + LeaseRead = buscompose.ControllerLease +) + +// The sources the switch names. +const ( + SourceInterim = "interim" + SourceIndexer = "indexer" + SourceController = "controller" +) + +// TakeOverVerb and HandBackVerb are the seat's verbs. +const ( + TakeOverVerb = "take-over" + HandBackVerb = "hand-back" + CompareVerb = "compare" +) + +// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask. +type Switch struct { + Source string `json:"source"` + At time.Time `json:"at"` + By string `json:"by"` + Why string `json:"why"` + Ask string `json:"ask,omitempty"` +} + +// source is the source that holds now. +func (c *Config) source() (Switch, error) { + e, err := c.Store.Get(TakeoverState, SwitchKey) + if err != nil { + return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err) + } + if e == nil { + return Switch{Source: SourceInterim}, nil + } + var s Switch + if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) { + return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value) + } + return s, nil +} + +// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading). +func (c *Config) leaseAway() bool { + e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +} + +// ---- declared -------------------------------------------------------------------------------------------- + +// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.` +// names holds them, with that manifest's digest and commit (ADR 0565 §3). +type DeclaredKept struct { + Module string `json:"module"` + Manifest string `json:"manifest"` + Commit string `json:"commit,omitempty"` + Config map[string]Declaration `json:"settings"` +} + +// trunkKept is a module's declarations at the trunk with the manifest they come from. +func (c *Config) trunkKept(module string) (DeclaredKept, error) { + key, err := delivery.ModulesKey(module) + if err != nil { + return DeclaredKept{}, err + } + e, err := c.Store.Get(ModulesRead, key) + if err != nil || e == nil { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err) + } + var m struct { + Manifest string `json:"manifest"` + Commit string `json:"commit"` + } + if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module) + } + d, err := c.manifestDeclarations(module, m.Manifest) + if err != nil { + return DeclaredKept{}, err + } + return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil +} + +// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs +// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written. +func (c *Config) writeDeclared(s Sources) ([]string, error) { + held := map[string][]byte{} + if err := c.each(DeclaredState, func(key string, raw []byte) error { + held[key] = raw + return nil + }); err != nil { + return nil, err + } + var written []string + var problems []error + placed := map[string]bool{} + for _, module := range modulesOf(s.Assignments) { + placed[module] = true + k, err := c.trunkKept(module) + if err != nil { + continue // unresolved, said by the resolution; its key stays as it was + } + raw, _ := json.Marshal(k) + if bytes.Equal(raw, held[module]) { + continue + } + if err := c.Store.Put(DeclaredState, module, k); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + continue + } + written = append(written, module) + } + for module := range held { + if !placed[module] { + if err := c.Store.Delete(DeclaredState, module); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + } + } + } + return written, errors.Join(problems...) +} + +// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source +// after a hand-back, and the other side of compare. +func (c *Config) mirrorDeclarations(s *Sources) { + s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil + for _, module := range modulesOf(s.Assignments) { + e, err := c.Store.Get(MirrorDeclaredRead, module) + var d ModuleDeclarations + if err == nil && e != nil { + d, err = ParseModuleDeclarations(e.Value) + } else if err == nil { + err = errors.New("the mirror holds no key for it") + } + if err != nil { + for _, a := range s.Assignments { + if a.Module == module { + s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module, + Why: "the controller's mirror declared-config: " + err.Error()}) + } + } + continue + } + s.Declared[module] = d + } +} + +// ---- the predefined list and the pull requests from the indexer ------------------------------------------ + +// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit. +func (c *Config) indexerList() (PredefinedList, bool, error) { + e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var kept delivery.PredefinedKept + if err := json.Unmarshal(e.Value, &kept); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err) + } + list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit} + if err := json.Unmarshal(kept.List, &list.Keys); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err) + } + return list, true, nil +} + +// mirrorList is the controller's mirror's list. +func (c *Config) mirrorList() (PredefinedList, bool, error) { + e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var list PredefinedList + if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { + return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value) + } + return list, true, nil +} + +// candidates is every open pull request's config declarations as the indexer keeps them, by pull key. +func (c *Config) candidates() (map[string]PullDecl, error) { + out := map[string]PullDecl{} + err := c.each(CandidatesRead, func(key string, raw []byte) error { + var cand delivery.ConfigCandidate + if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" { + return fmt.Errorf("the indexer's config candidate %s is not one", key) + } + p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At} + for module, keys := range cand.Declares { + raw, _ := json.Marshal(keys) + var d map[string]Declaration + if err := json.Unmarshal(raw, &d); err != nil { + return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module) + } + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } + d[k] = decl + } + p.Declares[module] = d + } + out[pullKey(cand.Pull)] = p + return nil + }) + return out, err +} + +// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one +// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502 +// §6). The caller holds the lock. +func (c *Config) takeCandidates() []string { + cands, err := c.candidates() + if err != nil { + return []string{"the indexer's config candidates cannot be read: " + err.Error()} + } + var s Sources + if err := c.readPulls(&s); err != nil { + return []string{err.Error()} + } + var said []string + for _, k := range sortedKeys(cands) { + p := cands[k] + if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) { + continue + } + a, err := c.declarePull(p.Pull, p.Head, p.Declares) + if err != nil { + said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err)) + continue + } + said = append(said, a.Said+" (from the indexer)") + } + return said +} + +// declaresDigest is what a pull request's declarations hold a value to, in one encoding. +func declaresDigest(d map[string]map[string]Declaration) string { + flat := map[string]string{} + for m, keys := range d { + for k, decl := range keys { + flat[m+"."+k] = declDigest(decl) + } + } + raw, _ := json.Marshal(flat) + return string(raw) +} + +// ---- compare ---------------------------------------------------------------------------------------------- + +// Difference is one thing the own sources and the controller's mirrors say differently, and why. +type Difference struct { + What string `json:"what"` + Module string `json:"module,omitempty"` + Node string `json:"node,omitempty"` + Key string `json:"key,omitempty"` + Pull string `json:"pull,omitempty"` + Own json.RawMessage `json:"own,omitempty"` + Mirror json.RawMessage `json:"mirror,omitempty"` + Why string `json:"why"` +} + +// The kinds of difference. +const ( + DiffDeclared = "declared" + DiffPredefined = "predefined" + DiffPull = "pull" + DiffValue = "value" +) + +// Comparison is what compare answers. +type Comparison struct { + Source string `json:"source"` + Differences []Difference `json:"differences"` + Count int `json:"count"` + Said string `json:"said"` +} + +// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock. +func (c *Config) Compare() (*Comparison, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.compare() +} + +// compare is Compare; the caller holds the lock. +func (c *Config) compare() (*Comparison, error) { + sw, err := c.source() + if err != nil { + return nil, err + } + own, err := c.sourcesFrom(SourceInterim) + if err != nil { + return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err) + } + mirror := own + mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil + c.mirrorDeclarations(&mirror) + var diffs []Difference + + // Each placed module's declarations: `declared` against the mirror's key for it. + for _, module := range modulesOf(own.Assignments) { + var ownRaw []byte + if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil { + var k DeclaredKept + if json.Unmarshal(e.Value, &k) == nil { + ownRaw, _ = json.Marshal(normal(k.Config)) + } + } + var mirrorRaw []byte + if d, ok := mirror.Declared[module]; ok { + mirrorRaw, _ = json.Marshal(normal(d.Config)) + } + switch { + case ownRaw == nil && mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"}) + case ownRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw, + Why: "the mirror holds its declarations and declared does not"}) + case mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, + Why: "declared holds its declarations and the mirror does not"}) + case !bytes.Equal(ownRaw, mirrorRaw): + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw, + Why: "the trunk's manifest declares it otherwise than the mirror"}) + } + } + + // The predefined list: the indexer's against the mirror's. + il, iok, ierr := c.indexerList() + ml, mok, merr := c.mirrorList() + switch { + case ierr != nil || merr != nil: + diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()}) + case !iok || !mok: + if iok != mok { + diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+ + "mirror holds one: %v", iok, mok)}) + } + default: + for _, k := range unionKeys(il.Keys, ml.Keys) { + a, aok := il.Keys[k] + b, bok := ml.Keys[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"}) + case !bytes.Equal(ar, br): + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br, + Why: "the two lists declare it otherwise"}) + } + } + } + + // Each open pull request: the indexer's candidate against what the controller told by pull-declares. + cands, cerr := c.candidates() + if cerr != nil { + diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()}) + } + for _, k := range unionKeys(cands, own.Pulls) { + a, aok := cands[k] + b, bok := own.Pulls[k] + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"}) + case a.Head != b.Head: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+ + "the controller told %s", short(a.Head), short(b.Head))}) + case declaresDigest(a.Declares) != declaresDigest(b.Declares): + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"}) + } + } + + // Every placed module's value on every node, resolved both ways. + ownValues, mirrorValues := Resolve(own), Resolve(mirror) + for _, k := range unionKeys(ownValues, mirrorValues) { + a, aok := ownValues[k] + b, bok := mirrorValues[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + node, module, key := a.Node, a.Module, a.Key + if !aok { + node, module, key = b.Node, b.Module, b.Key + } + d := Difference{What: DiffValue, Node: node, Module: module, Key: key} + switch { + case !aok: + d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node" + case !bok: + d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it" + case !bytes.Equal(ar, br): + d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source" + default: + continue + } + diffs = append(diffs, d) + } + for _, u := range mirror.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why}) + } + for _, u := range own.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why}) + } + sort.SliceStable(diffs, func(i, j int) bool { + a, b := diffs[i], diffs[j] + return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull + }) + if diffs == nil { + diffs = []Difference{} + } + cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)} + if len(diffs) == 0 { + cmp.Said = "nothing differs: the take-over may be asked" + } else { + cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs)) + } + return cmp, nil +} + +// normal is declarations in one encoding: a null default as none. +func normal(d map[string]Declaration) map[string]Declaration { + out := map[string]Declaration{} + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } else { + decl.Default = compact(decl.Default) + } + out[k] = decl + } + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// ---- take-over and hand-back --------------------------------------------------------------------------------- + +// TakeoverAsk is one take-over asked of the operator, `takeover.ask.`: what the ask is composed from again, and +// where it stands. +type TakeoverAsk struct { + ID string `json:"id"` + From string `json:"from"` + Why string `json:"why"` + At time.Time `json:"at"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Decided time.Time `json:"decided,omitzero"` + Answered string `json:"answered,omitempty"` + Recorded bool `json:"recorded,omitempty"` +} + +func askKey(id string) string { return "ask." + id } + +// act is what an answer to the take-over binds. +func (t TakeoverAsk) act(option string) asks.Act { + return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From, + "at": t.At.UTC().Format(time.RFC3339Nano)} +} + +// ask is the take-over's ask, composed from it alone. +func (t TakeoverAsk) ask() asks.Ask { + why, _ := sayable("Why: " + t.Why) + a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources", + Explanation: strings.Join([]string{ + "mesh-config would read the list of predefined keys and each pull request's config from the delivery's " + + "indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.", + point(why)}, "\n"), + Who: asks.Operator, Expires: t.At.Add(ProposalLasts), + OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover", + Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " + + "to the controller's records until the controller's lease is handed away."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"}, + {OptionDecline, "Decline", "nothing changes"}, + } { + binds, _ := asks.ActDigest(t.act(o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// TakeoverAnswer is what take-over and hand-back answer. +type TakeoverAnswer struct { + Said string `json:"said"` + Ask string `json:"ask,omitempty"` + Source string `json:"source"` +} + +// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference. +func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why the take-over is asked now: it is shown to the operator") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + if c.Asks == nil { + return nil, errors.New("the operator cannot be asked from here, so nothing is switched") + } + c.mu.Lock() + defer c.mu.Unlock() + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceIndexer { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " + + sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil + } + cmp, err := c.compare() + if err != nil { + return nil, err + } + if cmp.Count > 0 { + first := cmp.Differences[0] + return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+ + "Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why) + } + now := c.now() + var open []string + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting && + now.Before(t.At.Add(ProposalLasts)) { + open = append(open, t.ID) + } + return nil + }) + if len(open) > 0 { + return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0]) + } + t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(), + State: ProposalWaiting} + ask := t.ask() + if err := ask.Check(now); err != nil { + return nil, err + } + if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil { + return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err) + } + if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil { + t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error() + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return nil, fmt.Errorf("the operator could not be asked: %w", err) + } + return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+ + "mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+ + "then", ask.Headline)}, nil +} + +// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock. +func (c *Config) decideTakeovers() []string { + var said []string + var waiting []TakeoverAsk + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting { + waiting = append(waiting, t) + } + return nil + }) + now := c.now() + for _, t := range waiting { + keep := func(state, outcome string) string { + t.State, t.Outcome, t.Decided = state, outcome, now + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return "take-over " + t.ID + ": " + outcome + } + e, err := c.Asks.Record(t.ID) + if err != nil || e == nil { + if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) { + said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched")) + } + continue + } + var rec routerRecord + if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != t.ID { + said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched")) + continue + } + if w.Outcome != asks.OutcomeChosen { + said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched")) + continue + } + option, err := w.For(Module, t.ask()) + if err == nil { + err = option.Performs(t.act(option.ID)) + } + if err != nil { + said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error())) + continue + } + t.Answered = w.Says() + if option.ID != OptionApprove { + said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched")) + continue + } + cmp, err := c.compare() + switch { + case err != nil: + said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error())) + continue + case cmp.Count > 0: + said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+ + "now; nothing was switched, ask again once it is empty", cmp.Count))) + continue + } + sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID} + if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil { + said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+ + "on the next look", t.ID, err)) + continue + } + if err := c.Asks.Warranted(t.ID); err == nil { + t.Recorded = true + } + said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now")) + } + return said +} + +func approvedTakeover(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) +} + +// HandBack returns to the controller's records until the controller's lease is handed away. +func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why config hands back: it is kept with the switch") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + c.mu.Lock() + defer c.mu.Unlock() + if c.leaseAway() { + return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " + + "(novox/hq ADR 0565 §4)") + } + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceController { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil + } + now := c.now() + back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)} + if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil { + return nil, fmt.Errorf("the switch could not be written: %w", err) + } + answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " + + "declared-config and predefined, and pull-declares"} + if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil { + answer.Said += "; the hand-act log does not hold it: " + err.Error() + } + return answer, nil +} + +// Declared is the read of `declared`: every module's key, or one whole. +func (c *Config) Declared(module string) (any, error) { + if module != "" { + e, err := c.Store.Get(DeclaredState, module) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module) + } + return json.RawMessage(e.Value), nil + } + rows := map[string]any{} + err := c.each(DeclaredState, func(key string, raw []byte) error { + var k DeclaredKept + if json.Unmarshal(raw, &k) == nil { + rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)} + } + return nil + }) + sw, _ := c.source() + return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err +} + +// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and +// after the take-over the indexer's config candidates (ADR 0565 §15). +func (c *Config) Resync() (any, error) { + r, err := c.Reconcile() + if err != nil { + return r, err + } + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]any{"resolution": r} + if sw, err := c.source(); err == nil && sw.Source == SourceIndexer { + out["candidates"] = c.takeCandidates() + } + return out, nil +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover_test.go b/modules/mesh-config/cmd/mesh-config/takeover_test.go new file mode 100644 index 00000000..c66b21f8 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations. +const ( + claudeDeclared = `{"module":"claude-code","settings":{ + "managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}` + oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}` +) + +// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's +// declarations of every placed module, and the indexer's predefined list equal to the mirror's. +func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) { + t.Helper() + c, m, r := aProposingMesh(t) + m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared) + m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared) + m.mirror(t, MirrorDeclaredRead, "old", oldDeclared) + indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + return c, m, r +} + +// indexerList is the indexer's `predefined.list` at a commit. +func indexerList(t *testing.T, m *memStore, commit, list string) { + t.Helper() + raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)}) + m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw)) +} + +// candidate is the indexer's config candidate of a pull request. +func candidate(t *testing.T, m *memStore, pull, head, declares string) { + t.Helper() + key, err := delivery.ConfigCandidateKey(pull) + if err != nil { + t.Fatal(err) + } + m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`) +} + +func kinds(cmp *Comparison) []string { + var out []string + for _, d := range cmp.Differences { + k := d.What + ":" + d.Module + d.Pull + d.Key + if d.Node != "" { + k += "@" + d.Node + } + if !slices.Contains(out, k) { + out = append(out, k) + } + } + return out +} + +// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's +// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and +// its insight of 2026-10-29; issue 1953 L5-C4). +func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) { + cases := []struct { + name string + change func(t *testing.T, m *memStore) + want []string + }{ + {"all agree", func(*testing.T, *memStore) {}, nil}, + {"the mirror declares a default otherwise", func(t *testing.T, m *memStore) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + }, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}}, + {"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") }, + []string{"declared:old", "value:old@g14", "value:oldworkers@g14"}}, + {"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) { + m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1)) + }, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558) + {"the lists differ", func(t *testing.T, m *memStore) { + indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + }, []string{"predefined:agent-managed-settings"}}, + {"a pull request only the indexer holds", func(t *testing.T, m *memStore) { + candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + }, []string{"pull:novox/mesh-catalog#9"}}, + } + for _, tc := range cases { + c, m, r := anAgreeingMesh(t) + tc.change(t, m) + cmp, err := c.Compare() + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if got := kinds(cmp); !slices.Equal(got, tc.want) { + t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences) + } + for _, d := range cmp.Differences { + if d.Why == "" { + t.Errorf("%s: a difference without its reason: %+v", tc.name, d) + } + } + a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"}) + if len(tc.want) > 0 { + if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 { + t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err) + } + continue + } + if err != nil || a.Ask == "" || len(r.subjects) != 1 { + t.Errorf("%s: the take-over was not asked: %v", tc.name, err) + } + } +} + +// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests' +// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the +// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13). +func TestTheTakeOverAndTheHandBack(t *testing.T) { + c, m, r := anAgreeingMesh(t) + steps := []struct { + name string + do func() (string, error) + want string // what the answer or the error says + }{ + {"declared is written when it differs, and once", func() (string, error) { + _ = m.Delete(DeclaredState, "dunst") + first, err := c.Reconcile() + again, _ := c.Reconcile() + return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err + }, "dunst|"}, + {"pull-declares taken before the take-over", func() (string, error) { + a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0", + Declares: map[string]any{}}) + return answered(a), err + }, "keeps what"}, + {"the take-over is asked", func() (string, error) { + _ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate + a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"}) + return answered(a), err + }, "is asked"}, + {"the operator approves: switched to the indexer", func() (string, error) { + r.choose(t, r.lastAsk(t), OptionApprove, nil) + out, err := c.Decide() + return strings.Join(out, "; "), err + }, "reads its own sources from now"}, + {"pull-declares refused after it, naming the indexer", func() (string, error) { + _, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1", + Declares: map[string]any{}}) + return "", err + }, "config-candidates"}, + {"the list comes from the indexer", func() (string, error) { + indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72}, + "agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "theme":{"type":"string","description":"the desktop's theme"}}`) + res, err := c.Reconcile() + return res.Published, err + }, "theme"}, + {"a pull request's config comes from the indexer", func() (string, error) { + candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + res, err := c.Reconcile() + return strings.Join(res.Candidates, "; "), err + }, "from the indexer"}, + {"hand-back returns to the mirrors", func() (string, error) { + a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"}) + return answered(a), err + }, "controller's records again"}, + {"after it, declarations resolve from the mirror", func() (string, error) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + _, err := c.Reconcile() + return string(m.resolved(t, "g14", "dunst", "width").Value), err + }, "400"}, + {"hand-back is gone after the lease write", func() (string, error) { + m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`) + _, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"}) + return "", err + }, "gone"}, + {"the terminal sets no trusted value after the lease write", func() (string, error) { + _, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{}, + Why: "x", Caller: ControllerCaller, By: "person.jochen"}) + return "", err + }, "propose"}, + } + for _, s := range steps { + got, err := s.do() + text := got + if err != nil { + text = err.Error() + } + if !strings.Contains(text, s.want) { + t.Errorf("%s: said %q, want %q", s.name, text, s.want) + } + } + if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") { + t.Errorf("the hand-back is not recorded: %v", m.handActs) + } + if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) { + t.Errorf("declared holds no dunst with its manifest: %v", e) + } + // An Approve that finds compare differing at the answer switches nothing. + c2, m2, r2 := anAgreeingMesh(t) + if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil { + t.Fatal(err) + } + m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + r2.choose(t, r2.lastAsk(t), OptionApprove, nil) + out, _ := c2.Decide() + if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") { + t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out) + } +} + +// lastAsk is the newest ask the router took. +func (r *standIn) lastAsk(t *testing.T) string { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + var newest string + var at time.Time + for id, a := range r.asks { + if newest == "" || a.Expires.After(at) { + newest, at = id, a.Expires + } + } + if newest == "" { + t.Fatal("the router took no ask") + } + return newest +} + +func answered(a any) string { + raw, _ := json.Marshal(a) + return string(raw) +} diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 6a0791d7..a473279f 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,4 +2,13 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 8c9ba7ca..fe45f51a 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,2 +1,26 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 66db5b60..66641a25 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -23,6 +23,8 @@ "gitea.forge.pull-request.closed" ], "invokes": [ + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", "seat:mesh-controller.hand-act", "seat:mesh-controller.warranted", "gitea.gitea_get_pull_request" @@ -33,7 +35,18 @@ "mesh-delivery-builder.builds", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.manifests", - "mesh-controller.predefined" + "mesh-controller.predefined", + "mesh-controller.declared-config", + "mesh-delivery-indexer.predefined", + "mesh-delivery-indexer.config-candidates", + "mesh-controller.lease" + ], + "tools": [ + "config_compare", + "config_take_over", + "config_hand_back", + "config_declared", + "config_resync" ], "state": [ "values", @@ -41,7 +54,9 @@ "predefined", "proposals", "pulls", - "kept" + "kept", + "declared", + "takeover" ], "logs": [ "history" From 45db0e650e3163d86977542907f86cd907911277 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:27:23 +0100 Subject: [PATCH 14/20] Record a drill in mesh-hand-acts and count S15 over both logs, so the healers no longer refuse a drill (issue 1953) --- modules/mesh-healers/README.md | 5 +- .../mesh-healers/cmd/mesh-healers/healers.go | 3 + .../cmd/mesh-healers/healers_test.go | 65 ++++++++++++++++--- modules/mesh-healers/cmd/mesh-healers/main.go | 14 ++++ .../cmd/mesh-healers/manifest_test.go | 4 +- .../mesh-healers/cmd/mesh-healers/tools.go | 49 ++++++++++++-- .../mesh-healers/cmd/mesh-healers/wanted.go | 12 ++++ modules/mesh-healers/module.json | 7 +- 8 files changed, 137 insertions(+), 22 deletions(-) diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md index 30aac2c8..d00cdb1b 100644 --- a/modules/mesh-healers/README.md +++ b/modules/mesh-healers/README.md @@ -25,7 +25,7 @@ again, `fortnight` ends when S15's oldest counted act leaves the fortnight and c each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while no instance watched is judged at the start, once the lease is first read. -S15: it watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a +S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a fortnight; a drill and a person's decision by design never count. ## What it keeps @@ -41,5 +41,6 @@ fortnight; a drill and a person's decision by design never count. - `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why; `recount=true` counts the hand-act log again. - `mesh-healers.heal`: one act now on an open condition, by key and why. -- `mesh-healers.drill`: refused until the hand-act log's home is delivered (issue 1953), naming `mesh-controller.drill`. +- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the + controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill. - `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index d871de01..425eba47 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -27,6 +27,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -48,6 +49,8 @@ const ( ActedEvent = health.EventHealerActed // ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays). ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent + // RecordedEvent is the hand-act log's seat event, each act recorded there (ADR 0565 §11). + RecordedEvent = handacts.Seat + "." + handacts.EventRecorded // The events it hears: the keeper's, on the seat mesh-conditions. RaisedEvent = "mesh-conditions.condition-raised" ChangedEvent = "mesh-conditions.condition-changed" diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index 16ea85d3..9b6f9dce 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/handacts" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -379,9 +380,14 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { if keys := w.store.keys(FindingsState); len(keys) != 0 { t.Fatalf("one act is a finding: %v", keys) } - act("a2", "broker consumer-reset", "consumer-behind", 24*time.Hour) + // The second of the cause is the seat's own act, heard as act-recorded: one in each log counts two (ADR 0565 §11). + seatAct, _ := json.Marshal(HandAct{ID: "a2", At: w.now.Add(-24 * time.Hour), By: "person.jochen", + Verb: "broker consumer-reset", Why: "x", Cause: "consumer-behind"}) + w.h.ActRecorded(seatAct) + // A drill in each log counts none. act("a3", "hand-act drill", "drill", time.Hour) - act("a4", "hand-act drill", "drill", 2*time.Hour) + drill, _ := json.Marshal(HandAct{ID: "a4", At: w.now.Add(-2 * time.Hour), Verb: "hand-act drill", Why: "x", Cause: "drill"}) + w.h.ActRecorded(drill) act("a5", "conditions silence", "operator-answer", time.Hour) act("a6", "conditions silence", "operator-answer", time.Hour) act("a7", "hand-act record", "relogin", 20*24*time.Hour) @@ -402,7 +408,7 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { if keys := w.store.keys(FindingsState); len(keys) != 0 { t.Errorf("past the fortnight the finding stays: %v", keys) } - w.h.HandActHeard("a2", "delete", nil) + w.h.HandActHeard("a5", "delete", nil) if len(w.h.acts) != 7 { t.Errorf("a deleted entry is kept: %d", len(w.h.acts)) } @@ -436,14 +442,53 @@ func TestTheVerbsAndReads(t *testing.T) { if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 { t.Errorf("healers answered %v", a) } - var drill error - for _, tool := range Tools(w.h) { - if tool.Name == DrillTool { - _, drill = tool.Run(map[string]any{}) - } + // drill records into mesh-hand-acts as a drill; while that seat has no holder, into the controller's drill until + // the lease is handed away (ADR 0565 §11). + drills := []struct { + name string + seatGone bool + leaseAway bool + args map[string]any + log, err string + }{ + {"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""}, + {"no holder: the controller's", true, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogController, ""}, + {"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "handed its lease away"}, + {"says nothing", false, true, map[string]any{}, "", "says what"}, } - if drill == nil || !strings.Contains(drill.Error(), "mesh-controller.drill") { - t.Errorf("drill answered %v", drill) + for _, d := range drills { + x := newWorld() + if d.leaseAway { + x.handAway() + } else { + x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil) + } + var asked []string + x.h.Ask = func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if d.seatGone && strings.HasPrefix(key, "seat:mesh-hand-acts.") { + return nil, errors.New("nats: no responders available for request") + } + return json.RawMessage(`{}`), nil + } + var got any + var err error + for _, tool := range Tools(x.h) { + if tool.Name == DrillTool { + got, err = tool.Run(d.args) + } + } + switch { + case d.err != "": + if err == nil || !strings.Contains(err.Error(), d.err) { + t.Errorf("drill %s: %v", d.name, err) + } + case err != nil || got.(map[string]any)["log"] != d.log: + t.Errorf("drill %s: %v, %v", d.name, got, err) + case d.log == handacts.LogSeat && !strings.Contains(asked[0], `"verb":"hand-act drill"`): + t.Errorf("drill %s asked %v", d.name, asked) + } } if _, err := w.h.read(WouldState, ""); err != nil { t.Error(err) diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 7165f20b..6e48bfba 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -84,6 +84,20 @@ func main() { h.HandActHeard(c.Key, c.Op, c.Value) return nil }) + // S15 over both logs: the seat's act-recorded on a durable consumer, beside the controller's bucket's watch. + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(RecordedEvent, func(e stdio.Envelope) error { + h.ActRecorded(e.Body) + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, RecordedEvent, err, wait) + } + }() for _, event := range []string{RaisedEvent, ChangedEvent} { go func(event string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go index 08d16430..af3745b9 100644 --- a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -48,8 +48,8 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { got, want []string }{ {"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil}, - {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent}}, - {"invokes", m.Invokes, []string{AssertTool}}, + {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}}, + {"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-controller.drill"}}, {"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}}, } { if !slices.Equal(c.got, c.want) { diff --git a/modules/mesh-healers/cmd/mesh-healers/tools.go b/modules/mesh-healers/cmd/mesh-healers/tools.go index 3dd70607..d3452b95 100644 --- a/modules/mesh-healers/cmd/mesh-healers/tools.go +++ b/modules/mesh-healers/cmd/mesh-healers/tools.go @@ -13,6 +13,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" ) // Verbs are the seat's verbs, as the manifest claims them. @@ -55,12 +56,14 @@ func Tools(h *Healers) []stdio.Tool { return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a)) }}, {Name: DrillTool, - Description: "A drill: something broken on purpose to see the mesh raise and clear it, recorded in the hand-act " + - "log. Refused until the log's home is delivered (issue 1953): the controller's drill records it meanwhile.", - Input: obj(map[string]any{"why": map[string]any{"type": "string"}}), - Run: func(map[string]any) (any, error) { - return nil, errors.New("a drill is recorded in the hand-act log, whose one writer is the controller until its " + - "home is delivered (issue 1953): use mesh-controller.drill, which records it and S15 never counts") + Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with what it " + + "tests, in the hand-act log of the seat mesh-hand-acts (the controller's drill while that seat has no " + + "holder and the controller holds its lease). A drill is never a repair: S15 never counts it.", + Input: obj(map[string]any{"what": map[string]any{"type": "string", "description": "what was done on purpose"}, + "why": map[string]any{"type": "string", "description": "what the drill tests"}, + "condition": map[string]any{"type": "string", "description": "the condition it is meant to raise, if any"}}), + Run: func(a map[string]any) (any, error) { + return h.Drill(strArg(a, "what"), strArg(a, "why"), strArg(a, "condition")) }}, {Name: FindingsTool, Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " + @@ -81,6 +84,40 @@ func Tools(h *Healers) []stdio.Tool { } } +// Drill records a drill (ADR 0565 §11, which ends ADR 0564 Limits 1's refusal): through the seat mesh-hand-acts as +// the act `hand-act drill`, and while that seat has no holder that answers and the controller still holds its lease, +// through the controller's own `drill`, which records it as a drill too. It answers which log took it. +func (h *Healers) Drill(what, why, condition string) (any, error) { + if what == "" || why == "" { + return nil, errors.New("a drill says what was done on purpose and what it tests: nothing was recorded") + } + if h.Ask == nil { + return nil, errors.New("the hand-act log cannot be reached from here: nothing was recorded") + } + body := map[string]any{"what": what, "why": why, "verb": handacts.VerbDrill} + controller := map[string]any{"what": what, "why": why} + if condition != "" { + body["condition"], controller["condition"] = condition, condition + } + answer, err := h.Ask("seat:"+handacts.Seat+".hand-act", body) + if err == nil { + return map[string]any{"log": handacts.LogSeat, "answer": answer}, nil + } + if !handacts.NoResponder(err) { + return nil, fmt.Errorf("%s.hand-act refused the drill: %w", handacts.Seat, err) + } + if acting, _ := h.Acting(); acting { + return nil, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing was "+ + "recorded (%w)", handacts.Seat, err) + } + answer, cerr := h.Ask("seat:mesh-controller.drill", controller) + if cerr != nil { + return nil, fmt.Errorf("%s has no holder that answers (%v), and the controller's drill refused too: %w", handacts.Seat, + err, cerr) + } + return map[string]any{"log": handacts.LogController, "answer": answer}, nil +} + // Heal is the verb heal: one condition judged now by its healer. func (h *Healers) Heal(key, why, caller string) (any, error) { if key == "" || why == "" { diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go index 0c91821e..065f2c41 100644 --- a/modules/mesh-healers/cmd/mesh-healers/wanted.go +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -15,6 +15,8 @@ import ( "sort" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" ) // Within is how far back a repeated cause counts. @@ -66,6 +68,16 @@ func personsDecision(a HandAct) bool { return len(causes) == 0 || slices.Contains(causes, a.Cause) } +// ActRecorded is one act the seat mesh-hand-acts recorded, heard as act-recorded: counted beside the controller's +// log's, under a key of its own log, so an act is counted once whichever log holds it (ADR 0565 §11). +func (h *Healers) ActRecorded(body []byte) { + var a HandAct + if json.Unmarshal(body, &a) != nil || a.ID == "" { + return + } + h.HandActHeard(handacts.LogSeat+":"+a.ID, "put", body) +} + // HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again. func (h *Healers) HandActHeard(key, op string, value []byte) { h.mu.Lock() diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index 4319f46f..634dd388 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -15,10 +15,13 @@ ], "consumes": [ "mesh-conditions.condition-raised", - "mesh-conditions.condition-changed" + "mesh-conditions.condition-changed", + "mesh-hand-acts.act-recorded" ], "invokes": [ - "mesh-grants.grants_assert" + "mesh-grants.grants_assert", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-controller.drill" ], "reads": [ "mesh-controller.lease", From 4117ec3a577586ceb02396e4aa91fb1cdadc2318 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:29:48 +0100 Subject: [PATCH 15/20] Serve compare, take-over and hand-back as the mesh-config seat's verbs, beside the module's own tools; pin mesh-sdk 4204afd (issue 1953) --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 ++-- .../cmd/mesh-config/manifest_test.go | 20 ++++++++++++++---- .../cmd/mesh-config/switch_tools.go | 21 +++++++++++++++++-- modules/mesh-config/go.mod | 2 +- modules/mesh-config/go.sum | 4 ++-- modules/mesh-config/module.json | 5 ++++- modules/mesh-hand-acts/go.mod | 2 +- modules/mesh-hand-acts/go.sum | 4 ++-- modules/mesh-healers/go.mod | 2 +- modules/mesh-healers/go.sum | 4 ++-- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 ++-- 13 files changed, 54 insertions(+), 22 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index f9aedc80..937ab2e3 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f diff --git a/checks/health/go.sum b/checks/health/go.sum index 3725c41e..df962ed0 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go index 23f06ef3..c68914e4 100644 --- a/modules/mesh-config/cmd/mesh-config/manifest_test.go +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -16,10 +16,14 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { t.Fatal(err) } var m struct { - State []string `json:"state"` - Logs []string `json:"logs"` - Reads []string `json:"reads"` - Tools []string `json:"tools"` + State []string `json:"state"` + Logs []string `json:"logs"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -37,6 +41,14 @@ func TestEveryStateAndActHasItsTool(t *testing.T) { for _, tool := range Tools(&Config{}) { served[tool.Name] = true } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") { + t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs) + } + for _, verb := range SeatVerbs { + if !served[Seat+"."+verb] { + t.Errorf("the seat's verb %s is not served", verb) + } + } for _, name := range m.Tools { if !served[name] { t.Errorf("the manifest lists the tool %s, which is not served", name) diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go index 80983e4f..7fbc2ffc 100644 --- a/modules/mesh-config/cmd/mesh-config/switch_tools.go +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -1,7 +1,7 @@ package main -// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, the read of `declared` and -// resync. Module tools until mesh-sdk's seat mesh-config names them as its verbs. +// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs +// and as the module's own tools beside them, the read of `declared` and resync. import ( "strings" @@ -18,7 +18,24 @@ const ( ResyncTool = "config_resync" ) +// SeatVerbs are the seat's verbs, as the manifest claims them. +var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb} + func switchTools(c *Config) []stdio.Tool { + own := ownSwitchTools(c) + byName := map[string]stdio.Tool{} + for _, t := range own { + byName[t.Name] = t + } + for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} { + t := byName[tool] + t.Name = Seat + "." + verb + own = append(own, t) + } + return own +} + +func ownSwitchTools(c *Config) []stdio.Tool { why := map[string]any{"type": "object", "properties": map[string]any{ "why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}} whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) } diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index a473279f..596188af 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,7 +2,7 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f require ( github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index fe45f51a..a2e6f9cc 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 66641a25..9554db29 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -7,7 +7,10 @@ "name": "mesh-config", "scope": "mesh", "serves": [ - "config" + "config", + "compare", + "take-over", + "hand-back" ] } ], diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod index ae12ac5a..7de813c4 100644 --- a/modules/mesh-hand-acts/go.mod +++ b/modules/mesh-hand-acts/go.mod @@ -3,7 +3,7 @@ module mesh-hand-acts go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum index f53fc84a..89be360c 100644 --- a/modules/mesh-hand-acts/go.sum +++ b/modules/mesh-hand-acts/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index 84e2b59c..3986b5ca 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index f53fc84a..89be360c 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index 4f9316ba..aebf0142 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index f53fc84a..89be360c 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 077135a7f39289f8a684407522ad06484993d44c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:29 +0100 Subject: [PATCH 16/20] Keep mesh-config's held values in mesh-sdk's KeptValue layout, which the checks read, and pin mesh-sdk 66ff33d (issue 1953) The layout and its keys are unchanged, so no value moves. --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 +- modules/mesh-config/cmd/mesh-config/pulls.go | 41 ++++++++------------ modules/mesh-config/go.mod | 2 +- modules/mesh-config/go.sum | 4 +- modules/mesh-hand-acts/go.mod | 2 +- modules/mesh-hand-acts/go.sum | 4 +- modules/mesh-healers/go.mod | 2 +- modules/mesh-healers/go.sum | 4 +- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +- 11 files changed, 32 insertions(+), 39 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index 937ab2e3..13b76586 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d diff --git a/checks/health/go.sum b/checks/health/go.sum index df962ed0..383c583f 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go index df868157..8c0647a6 100644 --- a/modules/mesh-config/cmd/mesh-config/pulls.go +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -10,6 +10,8 @@ import ( "strconv" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826). @@ -61,31 +63,22 @@ type PullDecl struct { At time.Time `json:"at"` } -// KeptValue is an approved held value, kept apart from the values mesh-config resolves. -type KeptValue struct { - Pull string `json:"pull"` - Head string `json:"head"` - Digest string `json:"declaration"` - Node string `json:"node"` - Module string `json:"module"` - Key string `json:"key"` - Value json.RawMessage `json:"value"` - By string `json:"by"` - Via string `json:"via"` - Why string `json:"why"` - // Approved is who approved it, as the proposal kept it. - Approved string `json:"approved"` - Proposal string `json:"proposal"` - At time.Time `json:"at"` -} +// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the +// checker and the assigner's composition check read (novox/hq ADR 0565 §5). +type KeptValue = delivery.KeptValue +// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is +// not one. func keptKey(pull, node, module, key string) string { + if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil { + return k + } return pullKey(pull) + "." + node + "." + module + "." + key } -func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } +func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } -func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } +func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } // declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same. func declDigest(d Declaration) string { @@ -247,7 +240,7 @@ func (c *Config) declarePull(pull, head string, declares map[string]map[string]D // heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and // config-changed; what could not be done is answered, not dropped. func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string { - level := kv.level() + level := keptLevel(kv) ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why, Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}} unsaid := c.record(level.ValueKey(kv.Key), ch) @@ -264,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string // dropKept removes a held value, saying why. func (c *Config) dropKept(kv KeptValue, why string) []string { - if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil { - return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()} + if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil { + return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()} } c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why) return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "") @@ -420,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) { kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module, Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID, At: c.now()} - if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil { + if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil { return "", fmt.Errorf("the value was not kept: %v", err) } unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved) @@ -483,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string { said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...) continue } - vk := kv.level().ValueKey(kv.Key) + vk := keptLevel(kv).ValueKey(kv.Key) v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull + ", merged)", Approved: kv.Approved, At: c.now()} if err := c.Store.Put(ValuesState, vk, v); err != nil { diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 596188af..fd98e5ad 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,7 +2,7 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d require ( github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index a2e6f9cc..5ed89480 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod index 7de813c4..6727bf7c 100644 --- a/modules/mesh-hand-acts/go.mod +++ b/modules/mesh-hand-acts/go.mod @@ -3,7 +3,7 @@ module mesh-hand-acts go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum index 89be360c..326dffd4 100644 --- a/modules/mesh-hand-acts/go.sum +++ b/modules/mesh-hand-acts/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index 3986b5ca..e5cbdf2d 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index 89be360c..326dffd4 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index aebf0142..66a0e0d0 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 89be360c..326dffd4 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f h1:TbL5EGs7WCioK610IKPsEOn0mFx8mEjnvTbycaR1Alc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029212845-4204afd2400f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From 799f03fc2a4b8e3d58687b1a98ffc0149d9e1be7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:29 +0100 Subject: [PATCH 17/20] Refuse a manifest that records into the controller's hand-act log, naming the recorders' fallbacks until the lease write (issue 1953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0565 §11's catalog check: every recorder records through mesh-hand-acts. The fallbacks are listed with why, and the list only shrinks. --- checks/handacts/fallbacks | 8 +++ checks/handacts/go.mod | 3 + checks/handacts/main.go | 109 ++++++++++++++++++++++++++++++++++ checks/handacts/main_test.go | 59 ++++++++++++++++++ checks/handacts/reads-outside | 3 + 5 files changed, 182 insertions(+) create mode 100644 checks/handacts/fallbacks create mode 100644 checks/handacts/go.mod create mode 100644 checks/handacts/main.go create mode 100644 checks/handacts/main_test.go create mode 100644 checks/handacts/reads-outside diff --git a/checks/handacts/fallbacks b/checks/handacts/fallbacks new file mode 100644 index 00000000..c4fdf709 --- /dev/null +++ b/checks/handacts/fallbacks @@ -0,0 +1,8 @@ +# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each ` `: +# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until +# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any +# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write +# (novox/hq ADR 0565 §11; issue 1953, issue 774). +mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first +mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first +mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first diff --git a/checks/handacts/go.mod b/checks/handacts/go.mod new file mode 100644 index 00000000..29e4c52b --- /dev/null +++ b/checks/handacts/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/handacts + +go 1.22 diff --git a/checks/handacts/main.go b/checks/handacts/main.go new file mode 100644 index 00000000..1a222d60 --- /dev/null +++ b/checks/handacts/main.go @@ -0,0 +1,109 @@ +// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every +// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes +// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the +// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and +// an entry whose module no longer invokes the verb is refused too. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks. +var Verbs = []string{"hand-act", "warranted", "drill"} + +const ( + invokePrefix = "seat:mesh-controller." + Allowed = "checks/handacts/fallbacks" +) + +// Findings is every refusal in the catalog at root, sorted. +func Findings(root string) ([]string, error) { + allowed := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(root, Allowed)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]+" "+f[1]] = true + } else if len(f) > 0 && !strings.HasPrefix(f[0], "#") { + return nil, fmt.Errorf("%s: %q is not ` `", Allowed, line) + } + } + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var out []string + invoked := map[string]bool{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Invokes []string `json:"invokes"` + } + if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" { + continue + } + for _, inv := range m.Invokes { + verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix) + if !ok || !contains(Verbs, verb) { + continue + } + invoked[m.Module+" "+verb] = true + if !allowed[m.Module+" "+verb] { + out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+ + "or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed)) + } + } + } + for entry := range allowed { + if !invoked[entry] { + out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks", + Allowed, entry)) + } + } + sort.Strings(out) + return out, nil +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: handacts ") + os.Exit(2) + } + found, err := Findings(os.Args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } + fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named") +} diff --git a/checks/handacts/main_test.go b/checks/handacts/main_test.go new file mode 100644 index 00000000..61e197e6 --- /dev/null +++ b/checks/handacts/main_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is +// refused (novox/hq ADR 0565 §11's catalog test). +func TestFindings(t *testing.T) { + for _, row := range []struct { + name string + files map[string]string + want string // empty: clean + }{ + {"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""}, + {"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`}, + "a invokes seat:mesh-controller.warranted"}, + {"a named fallback, and the controller's own", map[string]string{ + "modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`, + "modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`, + Allowed: "# why\na drill fallback until the lease write\n"}, ""}, + {"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`}, + {"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""}, + } { + t.Run(row.name, func(t *testing.T) { + root := t.TempDir() + for path, text := range row.files { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := Findings(root) + if err != nil { + t.Fatal(err) + } + if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) { + t.Fatalf("findings = %q, want one containing %q", all, row.want) + } + }) + } +} + +// The run of this check over the real tree. +func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) { + got, err := Findings(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + for _, f := range got { + t.Error(f) + } +} diff --git a/checks/handacts/reads-outside b/checks/handacts/reads-outside new file mode 100644 index 00000000..61f9814a --- /dev/null +++ b/checks/handacts/reads-outside @@ -0,0 +1,3 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what it invokes. +modules/*/module.json From 0a2334e2cd11b523517086d6f7e4c71d09dbc69d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:29 +0100 Subject: [PATCH 18/20] Compose every provision and plan in mesh-provisioner from the bus on change events, its state from the providers' outcomes, and ask before retiring (issue 1954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seat holds provisions and plans alone (ADR 0570 §1-§3, §5): a node is composed at its wanted builds and its newest, so a key ahead of its want is never retiring; a key no build needs is retiring with one ask, and the warrant goes into the key for its provider. The registry's placement kinds name plans; needs is emptied at start. Pins mesh-sdk 66ff33d. --- modules/mesh-provisioner/.gitignore | 2 + .../cmd/mesh-provisioner/busread.go | 204 ++++++ .../cmd/mesh-provisioner/holder.go | 617 ++++++++++++++++++ .../cmd/mesh-provisioner/holder_test.go | 257 ++++++++ .../cmd/mesh-provisioner/holdertools.go | 104 +++ .../cmd/mesh-provisioner/main.go | 172 ++++- .../cmd/mesh-provisioner/provisioner_test.go | 42 +- .../cmd/mesh-provisioner/reads-outside | 2 + .../cmd/mesh-provisioner/registry.go | 15 +- modules/mesh-provisioner/go.mod | 2 +- modules/mesh-provisioner/go.sum | 16 +- modules/mesh-provisioner/module.json | 27 +- 12 files changed, 1420 insertions(+), 40 deletions(-) create mode 100644 modules/mesh-provisioner/.gitignore create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/busread.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holder.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside diff --git a/modules/mesh-provisioner/.gitignore b/modules/mesh-provisioner/.gitignore new file mode 100644 index 00000000..b51e2c5c --- /dev/null +++ b/modules/mesh-provisioner/.gitignore @@ -0,0 +1,2 @@ +/mesh-provisioner +/cmd/mesh-provisioner/mesh-provisioner diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/busread.go b/modules/mesh-provisioner/cmd/mesh-provisioner/busread.go new file mode 100644 index 00000000..5de3c58c --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/busread.go @@ -0,0 +1,204 @@ +package main + +// The bus as the holder composes from it (frombus.Reader), read through this module's runtime with its own `reads`: +// placement, each node's record in its own have key, the indexer's modules and manifests, the builder's builds and +// mesh-config's resolved values. A plan names no value, so no own secret and no credential is read (compose.Planning). + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// busReader reads the bus for frombus through a Store. +type busReader struct { + s Store + fetch func(ctx context.Context, url string) ([]byte, error) +} + +func (r busReader) each(read string, visit func(key string, value []byte)) error { + keys, err := r.s.Keys(read) + if err != nil { + return fmt.Errorf("%s: %w", read, err) + } + for _, k := range keys { + e, err := r.s.Get(read, k) + if err != nil { + return fmt.Errorf("%s %s: %w", read, k, err) + } + if e != nil { + visit(k, e.Value) + } + } + return nil +} + +func (r busReader) Placements(context.Context) (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + err := r.each(PlacementRead, func(k string, v []byte) { + node, err := delivery.ParsePlacementKey(k) + if err != nil { + return + } + if p, err := delivery.DecodePlacement(v); err == nil && p.Node == node { + out[node] = p + } + }) + return out, err +} + +func (r busReader) Records(context.Context) (map[string]delivery.NodeRecord, error) { + out := map[string]delivery.NodeRecord{} + err := r.each(HaveRead, func(k string, v []byte) { + node, _, isNode, err := delivery.ParseHaveKey(k) + if err != nil || !isNode { + return + } + if h, err := delivery.DecodeHaveNode(v); err == nil && h.Record != nil { + out[node] = *h.Record + } + }) + return out, err +} + +func (r busReader) Modules(context.Context) (map[string]delivery.Module, error) { + out := map[string]delivery.Module{} + err := r.each(ModulesRead, func(k string, v []byte) { + module, err := delivery.ParseModulesKey(k) + if err != nil { + return + } + if m, err := delivery.DecodeModule(v); err == nil { + out[module] = m + } + }) + return out, err +} + +func (r busReader) Manifest(_ context.Context, digest string) ([]byte, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return nil, err + } + e, err := r.s.Get(ManifestsRead, key) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("the indexer holds no manifest %s", digest) + } + m, err := delivery.DecodeManifest(e.Value) + if err != nil { + return nil, err + } + return []byte(m.JSON), nil +} + +func (r busReader) Values(_ context.Context, node, module string) (map[string]any, error) { + var out map[string]any + prefix := node + "." + module + "." + err := r.each(ResolvedRead, func(k string, v []byte) { + name, ok := strings.CutPrefix(k, prefix) + if !ok || strings.Contains(name, ".") { + return + } + if value, laid := catalogue.LaidValue(v); laid { + if out == nil { + out = map[string]any{} + } + out[name] = value + } + }) + return out, err +} + +func (busReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { return nil, nil } + +func (busReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + return nil, nil, nil +} + +func (r busReader) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return nil, err + } + e, err := r.s.Get(BuildsRead, key) + if err != nil || e == nil { + return nil, err + } + b, err := delivery.DecodeBuilderBuild(e.Value) + if err != nil { + return nil, err + } + return &b, nil +} + +// archiveWithin bounds one read of a templates archive. +const archiveWithin = 30 * time.Second + +func (r busReader) Archive(ctx context.Context, url, digest string) ([]byte, error) { + if r.fetch == nil { + return nil, fmt.Errorf("this holder reads no archive (%s)", url) + } + fetching, cancel := context.WithTimeout(ctx, archiveWithin) + defer cancel() + body, err := r.fetch(fetching, url) + if err != nil { + return nil, err + } + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got) + } + return body, nil +} + +func (r busReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) { + all, err := r.Values(ctx, node, module) + var out map[string]any + for _, k := range keys { + if v, ok := all[k]; ok { + if out == nil { + out = map[string]any{} + } + out[k] = v + } + } + return out, err +} + +// httpFetch reads an archive from the artifact store over HTTP. +func httpFetch(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s", url, resp.Status) + } + return io.ReadAll(io.LimitReader(resp.Body, 256<<20)) +} + +// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names. +func FromBus(s Store) Composer { + r := busReader{s: s, fetch: httpFetch} + return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go new file mode 100644 index 00000000..04c2fb3e --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -0,0 +1,617 @@ +package main + +// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed +// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them +// in its states `plans` (`.`) and `provisions` (`..[.]`), which it alone +// writes. +// +// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the +// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants +// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a +// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5). +// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that +// consumer, `_bound` under the same key: present is provided (the vault's generation carried into the +// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by +// itself, but for a provider that keeps no entries at all (its manifest declares no `bound`), which makes nothing per +// consumer. +// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's +// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it; +// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The states it keeps, beside Registry, Needs, Applied and Collect. +const ( + ProvisionsState = provisions.ProvisionsState + PlansState = provisions.PlansState +) + +// The reads it composes from. +const ( + PlacementRead = delivery.PlacementRead + WantsRead = delivery.AssignerModule + ".wants" + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" + BuildsRead = delivery.BuilderModule + ".builds" + HaveRead = "node-engine.have" + ResolvedRead = "mesh-config.resolved" +) + +// Providers are the providing modules that keep an entry per consumer, `_bound`, read as `.bound`: the +// outcome of each provision they make. A provider not named here keeps none. +var Providers = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"} + +// boundRead is a provider's entries as this module reads them. +func boundRead(module string) string { return module + ".bound" } + +// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names +// (module → fingerprint), the newest build of every other placed module. +type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) + +// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's +// record of it. +type Asker interface { + Publish(subject string, body any, id string) error + Record(id string) ([]byte, bool, error) +} + +// Holder is the seat's holder. +type Holder struct { + Store Store + Compose Composer + Asks Asker + Now func() time.Time + Logf func(format string, args ...any) + + mu sync.Mutex +} + +// Composed is what one node's composition did. +type Composed struct { + Node string `json:"node"` + Plans []string `json:"plans-written,omitempty"` + Gone []string `json:"plans-removed,omitempty"` + Changed []string `json:"provisions-changed,omitempty"` + Asked []string `json:"retiring-asked,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +// ComposeNode composes one node: its plans and its provisions. +func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) { + h.mu.Lock() + defer h.mu.Unlock() + out := Composed{Node: node} + wanted, err := h.wantedOn(node) + if err != nil { + return out, err + } + atWants, err := h.Compose(ctx, node, wanted) + if err != nil { + return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) + } + newest, err := h.Compose(ctx, node, nil) + if err != nil { + return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err) + } + if err := h.writePlans(node, atWants, &out); err != nil { + out.Problems = append(out.Problems, err.Error()) + } + needed := map[string]provisions.Provision{} + for _, plans := range []map[string]catalogue.Plan{atWants, newest} { + for module, plan := range plans { + for _, p := range plan.Provisions { + pv := provisions.Provision{Provision: p.Provision, Local: p.Local, + Consumer: provisions.End{Module: module, Node: node}, + Provider: provisions.End{Module: p.ProviderModule, Node: p.Provider}} + key, err := pv.Key() + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if _, seen := needed[key]; !seen { + needed[key] = pv + } + } + } + } + h.judgeNode(node, needed, &out) + return out, nil +} + +// wantedOn is the fingerprint each want on a node names, by module. +func (h *Holder) wantedOn(node string) (map[string]string, error) { + out := map[string]string{} + keys, err := h.Store.Keys(WantsRead) + if err != nil { + return nil, fmt.Errorf("the wants cannot be read: %w", err) + } + for _, k := range keys { + n, module, err := delivery.ParseWantKey(k) + if err != nil || n != node { + continue + } + e, err := h.Store.Get(WantsRead, k) + if err != nil || e == nil { + continue + } + if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" { + out[module] = w.FP + } + } + return out, nil +} + +// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. +func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { + var problems []error + for _, module := range sortedKeys(plans) { + key, err := provisions.PlanKey(node, module) + if err != nil { + problems = append(problems, err) + continue + } + raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]}) + if err != nil { + problems = append(problems, err) + continue + } + if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) { + continue + } + if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil { + problems = append(problems, fmt.Errorf("the plan %s: %w", key, err)) + continue + } + out.Plans = append(out.Plans, key) + } + keys, err := h.Store.Keys(PlansState) + if err != nil { + return errors.Join(append(problems, err)...) + } + for _, k := range keys { + n, module, ok := strings.Cut(k, ".") + if !ok || n != node { + continue + } + if _, still := plans[module]; still { + continue + } + if err := h.Store.Delete(PlansState, k); err != nil { + problems = append(problems, err) + continue + } + out.Gone = append(out.Gone, k) + } + return errors.Join(problems...) +} + +// judgeNode brings the node's provision keys to what is needed and what each provider said. +func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) { + held := h.held(node) + for _, key := range unionKeys(needed, held) { + want, isNeeded := needed[key] + have, isHeld := held[key] + var next provisions.Provision + switch { + case isNeeded: + next = want + if isHeld { + next.Reference = have.Reference + } + h.outcome(key, &next) + case have.State == provisions.Retiring: + next = have + if next.Ask == "" && next.Warrant == nil { + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if gone, why := h.retired(key, have); gone { + if err := h.Store.Delete(ProvisionsState, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Changed = append(out.Changed, key+" (retired: "+why+")") + } + continue + } + default: + next = have + next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+ + "any more", node, have.Provision, have.Provider.Module, have.Provider.Node) + next.Warrant = nil + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if isHeld && same(have, next) { + continue + } + next.At = h.Now().UTC() + raw, err := provisions.Encode(next) + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err)) + continue + } + out.Changed = append(out.Changed, key+" "+next.State) + } +} + +// held is the node's provision keys as kept. +func (h *Holder) held(node string) map[string]provisions.Provision { + out := map[string]provisions.Provision{} + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return out + } + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + if p, err := provisions.Decode(e.Value); err == nil { + out[k] = p + } + } + return out +} + +// same says two provisions say the same, their time aside. +func same(a, b provisions.Provision) bool { + a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0 + ra, _ := json.Marshal(a) + rb, _ := json.Marshal(b) + return bytes.Equal(ra, rb) +} + +// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what +// the holder needs and nothing more. +type entry struct { + Refused string `json:"refused,omitempty"` + Waits string `json:"waits,omitempty"` + VaultName string `json:"vault-name,omitempty"` + Generation int `json:"generation,omitempty"` +} + +// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has +// none. A provider that keeps no entries makes nothing per consumer: provided, said so. +func (h *Holder) outcome(key string, p *provisions.Provision) { + p.Ask, p.Warrant = "", nil + if !keepsEntries(p.Provider.Module) { + p.State, p.Why = provisions.Provided, p.Provider.Module+" makes nothing per consumer: what it provides is its "+ + "placement and served values" + return + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil { + p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "" + return + } + var en entry + if err := json.Unmarshal(e.Value, &en); err != nil { + p.State, p.Why = provisions.Asked, "the provider's entry is not readable" + return + } + switch { + case en.Refused != "": + p.State, p.Why = provisions.Failed, en.Refused + case en.Waits != "": + p.State, p.Why = provisions.Waits, en.Waits + default: + p.State, p.Why = provisions.Provided, "" + if en.VaultName != "" || en.Generation > 0 { + p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation} + } + } +} + +func keepsEntries(module string) bool { + for _, m := range Providers { + if m == module { + return true + } + } + return false +} + +// retired says a retiring key the operator approved is retired: its provider's entry is gone. +func (h *Holder) retired(key string, p provisions.Provision) (bool, string) { + if p.Warrant == nil || p.Warrant.Option != OptionApprove { + return false, "" + } + if !keepsEntries(p.Provider.Module) { + return true, "its provider keeps nothing for it" + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil || e != nil { + return false, "" + } + return true, "its provider removed its credential on the operator's warrant" +} + +// ---- the retiring ask ---------------------------------------------------------------------------------------------- + +// The options a retiring ask offers. +const ( + OptionApprove = "approve" + OptionDecline = "decline" +) + +// askLasts is how long a retiring ask stays open. +const askLasts = asks.ApproveLasts - 10*time.Minute + +// retiringAsk is the ask of one retiring provision, composed from the key alone. +func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask { + what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node, + p.Consumer.Module, p.Consumer.Node) + a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?", + Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+ + "is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node, + p.Provision, p.Provider.Module, what), + Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes", + About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"}, + {OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"}, + } { + binds, _ := asks.ActDigest(retireAct(key, id, o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +func retireAct(key, id, option string) asks.Act { + return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option} +} + +// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands. +func (h *Holder) ask(p *provisions.Provision, key string) error { + if p.Ask != "" { + return nil + } + if h.Asks == nil { + return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed") + } + now := h.Now().UTC() + id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix()) + a := retiringAsk(key, *p, now, id) + if err := a.Check(now); err != nil { + return err + } + if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil { + return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err) + } + p.Ask = id + return nil +} + +// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the +// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes. +func (h *Holder) Decide() []string { + h.mu.Lock() + defer h.mu.Unlock() + if h.Asks == nil { + return nil + } + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return []string{"the provisions cannot be read: " + err.Error()} + } + var said []string + for _, k := range keys { + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + p, err := provisions.Decode(e.Value) + if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil { + continue + } + raw, found, err := h.Asks.Record(p.Ask) + if err != nil || !found { + continue + } + var rec struct { + State string `json:"state"` + Ask *asks.Ask `json:"ask"` + Warrant *asks.Warrant `json:"warrant"` + } + if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != p.Ask { + continue + } + option := "" + if w.Outcome == asks.OutcomeChosen { + o, err := w.For(Module, *rec.Ask) + if err == nil { + err = o.Performs(retireAct(k, p.Ask, o.ID)) + } + if err != nil { + said = append(said, k+": the answer is refused: "+err.Error()) + continue + } + option = o.ID + } else { + // Ended with no choice: asked again at the next change. + p.Ask = "" + h.put(k, p) + said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes") + continue + } + by := "the operator" + if w.By != nil { + by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) + } + p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()} + h.put(k, p) + said = append(said, k+": "+option+" by "+by) + } + return said +} + +func (h *Holder) put(key string, p provisions.Provision) { + p.At = h.Now().UTC() + if raw, err := provisions.Encode(p); err == nil { + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + h.Logf("[%s] %s is not written: %v", Module, key, err) + } + } +} + +// ---- what a change touches -------------------------------------------------------------------------------------- + +// NodesOfPlacement is the node a placement key names. +func NodesOfPlacement(key string) []string { + if n, err := delivery.ParsePlacementKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfWant is the node a want key names. +func NodesOfWant(key string) []string { + if n, _, err := delivery.ParseWantKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change +// of the module's manifest touches. +func (h *Holder) NodesOfModule(module string) []string { + nodes := map[string]bool{} + if keys, err := h.Store.Keys(PlacementRead); err == nil { + for _, k := range keys { + n, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + e, err := h.Store.Get(PlacementRead, k) + if err != nil || e == nil { + continue + } + if p, err := delivery.DecodePlacement(e.Value); err == nil { + for _, m := range p.Modules { + if m.Module == module { + nodes[n] = true + } + } + } + } + } + if keys, err := h.Store.Keys(ProvisionsState); err == nil { + for _, k := range keys { + if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil { + if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module { + nodes[p.Consumer.Node] = true + } + } + } + } + return sortedKeys(nodes) +} + +// NodeOfEntry is the consumer's node a provider's entry key names. +func NodeOfEntry(key string) []string { + if n, _, _, _, err := provisions.ParseKey(key); err == nil { + return []string{n} + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A +// retiring key whose provider removed its entry on the operator's warrant goes. +func (h *Holder) Rejudge(key string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + e, err := h.Store.Get(ProvisionsState, key) + if err != nil || e == nil { + return "", err + } + have, err := provisions.Decode(e.Value) + if err != nil { + return "", err + } + if have.State == provisions.Retiring { + if gone, why := h.retired(key, have); gone { + return key + " retired: " + why, h.Store.Delete(ProvisionsState, key) + } + return "", nil + } + next := have + h.outcome(key, &next) + if same(have, next) { + return "", nil + } + h.put(key, next) + return key + " " + next.State, nil +} + +// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at +// the start. +func (h *Holder) RetireNeeds() (int, error) { + keys, err := h.Store.Keys(Needs) + if err != nil { + return 0, err + } + n := 0 + for _, k := range keys { + if err := h.Store.Delete(Needs, k); err != nil { + return n, err + } + n++ + } + return n, nil +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go new file mode 100644 index 00000000..1164f728 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -0,0 +1,257 @@ +package main + +import ( + "context" + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// router is the operator's channel as the holder asks on it. +type router struct { + asked map[string]asks.Ask + records map[string][]byte +} + +func (r *router) Publish(_ string, body any, id string) error { + r.asked[id] = body.(asks.Ask) + return nil +} + +func (r *router) Record(id string) ([]byte, bool, error) { + raw, ok := r.records[id] + return raw, ok, nil +} + +// answer is the operator's answer to an ask. +func (r *router) answer(t *testing.T, id, option string) { + t.Helper() + a := r.asked[id] + o, ok := a.Option(option) + if !ok { + t.Fatalf("the ask %s offers no %s", id, option) + } + w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, + Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram", + At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()} + r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w}) +} + +// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted +// builds and at the newest. +type mesh struct { + h *Holder + s *memStore + r *router + atWant map[string]map[string]catalogue.Plan + atNewest map[string]map[string]catalogue.Plan + composed []string +} + +func newMesh() *mesh { + m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}}, + atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}} + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + m.composed = append(m.composed, node) + if keptAt != nil { + return m.atWant[node], nil + } + return m.atNewest[node], nil + }} + return m +} + +// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set). +func planOf(provs ...catalogue.PlanProvision) catalogue.Plan { + return catalogue.Plan{Provisions: provs} +} + +var ( + pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"} + oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"} + split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"} +) + +func (m *mesh) state(t *testing.T, key string) provisions.Provision { + t.Helper() + e, _ := m.s.Get(ProvisionsState, key) + if e == nil { + return provisions.Provision{} + } + p, err := provisions.Decode(e.Value) + if err != nil { + t.Fatal(err) + } + return p +} + +func (m *mesh) compose(t *testing.T, node string) Composed { + t.Helper() + c, err := m.h.ComposeNode(context.Background(), node) + if err != nil { + t.Fatal(err) + } + return c +} + +// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry +// alone; a provider that keeps no entries makes nothing per consumer; the vault's generation reaches the reference; a +// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2). +func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { + m := newMesh() + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)} + m.atNewest["g14"] = m.atWant["g14"] + key := "g14.gitea.postgres-database" + steps := []struct { + name string + entry string // the provider's entry written before the step, "-" to delete + state string + why string + }{ + {"composed: asked", "", provisions.Asked, ""}, + {"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"}, + {"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"}, + {"the provider made it", `{"vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""}, + } + for _, s := range steps { + if s.entry != "" { + m.s.write(t, "postgres.bound", key, s.entry) + } + m.compose(t, "g14") + p := m.state(t, key) + if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" { + t.Errorf("%s: %+v", s.name, p) + } + } + if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 { + t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference) + } + if p := m.state(t, "g14.gitea.split-dns"); p.State != provisions.Provided || !strings.Contains(p.Why, "makes nothing") { + t.Errorf("a provider keeping no entries: %+v", p) + } + if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { + t.Error("the plan is not written") + } + if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 { + t.Errorf("a quiet pass wrote %+v", c) + } + // An entry's change judges its key alone, no composition. + before := len(m.composed) + m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`) + if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before { + t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before) + } +} + +// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed +// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the +// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again; +// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3). +func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { + m := newMesh() + pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client" + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want + m.s.write(t, "postgres.bound", pg, `{}`) + steps := []struct { + name string + change func() + asked []string + states map[string]string + removed bool + }{ + {"a key composed ahead of its want is asked, not retiring", func() {}, nil, + map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false}, + {"one requirement removed and another added: only the removed retires", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + }, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false}, + {"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false}, + {"needed again before the warrant: back to its outcome", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)} + }, nil, map[string]string{pg: provisions.Provided}, false}, + {"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} }, + []string{pg}, map[string]string{pg: provisions.Retiring}, false}, + } + for _, s := range steps { + s.change() + c := m.compose(t, "g14") + if !slices.Equal(c.Asked, s.asked) { + t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked) + } + for k, st := range s.states { + if p := m.state(t, k); p.State != st { + t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why) + } + } + } + p := m.state(t, pg) + a := m.r.asked[p.Ask] + if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") || + !strings.Contains(a.Explanation, "data stay") { + t.Errorf("the ask does not say what goes from which provider on which node: %+v", a) + } + // Nothing before the warrant: the provider's entry stands and the key stays. + if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil { + t.Errorf("decided before an answer: %v", said) + } + m.r.answer(t, p.Ask, OptionApprove) + m.h.Decide() + if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") { + t.Fatalf("the warrant is not in the key: %+v", w) + } + if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring { + t.Errorf("retired while the provider's entry stands: %q", said) + } + _ = m.s.Delete("postgres.bound", pg) + if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" { + t.Errorf("not retired once the provider removed its entry: %q", said) + } + // A decline is kept, and nothing is asked again while nothing changes. + d := newMesh() + d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + d.compose(t, "g14") + d.atWant["g14"] = map[string]catalogue.Plan{} + d.compose(t, "g14") + d.r.answer(t, d.state(t, pg).Ask, OptionDecline) + d.h.Decide() + if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline { + t.Errorf("a decline was asked again: %+v", c) + } +} + +// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1). +func TestTheHolderHoldsNoTimer(t *testing.T) { + files, _ := filepath.Glob("*.go") + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" && + slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) { + t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name) + } + } + return true + }) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go new file mode 100644 index 00000000..30a16446 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -0,0 +1,104 @@ +package main + +// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and +// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The holder's tools. +const ( + ProvisionsTool = "provisioner_provisions" + PlansTool = "provisioner_plans" + ComposeTool = "provisioner_compose" + DecideTool = "provisioner_decide" +) + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +func holderTools(h *Holder) []stdio.Tool { + arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: ProvisionsTool, + Description: "Every provision the seat keeps (novox/hq ADR 0570): its key ..[.], " + + "provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " + + "state, with how many keys and bytes the state holds; or one key whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"), + "node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}}, + Run: func(a map[string]any) (any, error) { + return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state")) + }}, + {Name: PlansTool, + Description: "Every module's provisioning plan the seat composed, ., narrowed by node, with how many " + + "keys and bytes the state holds; or one plan whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, ."), + "node": str("a node")}}, + Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }}, + {Name: ComposeTool, + Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " + + "was written, removed, changed and asked.", + Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")}, + "required": []string{"node"}}, + Run: func(a map[string]any) (any, error) { + if arg(a, "node") == "" { + return nil, fmt.Errorf("compose names the node") + } + return h.ComposeNode(context.Background(), arg(a, "node")) + }}, + {Name: DecideTool, + Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " + + "key, as the router's word on an ask would.", + Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }}, + } +} + +// read is a state's keys narrowed, or one whole, with the state's fill. +func (h *Holder) read(state, key, node, want string) (any, error) { + if key != "" { + e, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("%s holds no %s", state, key) + } + return json.RawMessage(e.Value), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := []map[string]any{} + bytes := 0 + for _, k := range keys { + e, err := h.Store.Get(state, k) + if err != nil || e == nil { + continue + } + bytes += len(e.Value) + if node != "" && !strings.HasPrefix(k, node+".") { + continue + } + row := map[string]any{"key": k} + if state == ProvisionsState { + p, err := provisions.Decode(e.Value) + if err != nil { + row["unreadable"] = err.Error() + } else { + if want != "" && p.State != want { + continue + } + row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why + } + } + rows = append(rows, row) + } + return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 3955516d..583072b0 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -1,23 +1,28 @@ // mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR -// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR -// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its -// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its -// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this -// module says, it says on stderr. +// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259 +// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change +// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and +// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its +// start. stdout is the MCP channel; what this module says, it says on stderr. package main import ( + "context" "fmt" "os" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. const ( - Seat = "mesh-provisioner" - Verb = "provisioning" + Seat = "mesh-provisioner" + Module = "mesh-provisioner" + Verb = "provisioning" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state). func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the +// router's record of one read under that name. +type runtimeAsks struct{} + +func (runtimeAsks) Publish(subject string, body any, id string) error { + return stdio.SeatPublish(subject, body, id) +} + +func (runtimeAsks) Record(id string) ([]byte, bool, error) { + e, err := stdio.SeatRecord("", id) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake. +type work struct { + mu sync.Mutex + pending map[string]bool + wake chan struct{} +} + +func (w *work) add(nodes ...string) { + if len(nodes) == 0 { + return + } + w.mu.Lock() + for _, n := range nodes { + w.pending[n] = true + } + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *work) run(h *Holder) { + for range w.wake { + w.mu.Lock() + nodes := sortedKeys(w.pending) + w.pending = map[string]bool{} + w.mu.Unlock() + for _, n := range nodes { + c, err := h.ComposeNode(context.Background(), n) + switch { + case err != nil: + logf("[%s] %s is not composed: %v", Module, n, err) + case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0: + logf("[%s] composed %s: %+v", Module, n, c) + } + } + } +} + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch("", handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + func main() { - go publishing(runtimeStore{}) - if err := stdio.Serve("", Tools(runtimeStore{})); err != nil { + s := runtimeStore{} + h := &Holder{Store: s, Compose: FromBus(s), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} + w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)} + go w.run(h) + go starting(h) + go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil }) + go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil }) + go watched(ModulesRead, func(c stdio.StateChange) error { + if module, err := delivery.ParseModulesKey(c.Key); err == nil { + w.add(h.NodesOfModule(module)...) + } + return nil + }) + for _, provider := range Providers { + go watched(boundRead(provider), func(c stdio.StateChange) error { + if said, err := h.Rejudge(c.Key); err != nil { + logf("[%s] %s is not judged again: %v", Module, c.Key, err) + } else if said != "" { + logf("[%s] %s", Module, said) + } + return nil + }) + } + go watched(Registry, func(c stdio.StateChange) error { + if c.Op != "put" { + if _, _, err := Publish(s); err != nil { + logf("[%s] the registry is not written back: %v", Module, err) + } + } + return nil + }) + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error { + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) + } + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait) + } + }() + if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil { logf("%v", err) os.Exit(1) } } -// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or -// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first -// pass is tried again until it lands. -func publishing(s Store) { - for wait := 2 * time.Second; ; { - time.Sleep(wait) - written, deleted, err := Publish(s) - switch { - case err != nil: - logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait) - wait = min(wait*2, time.Minute) - continue - case len(written) > 0 || len(deleted) > 0: - logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted) +// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers, +// each tried again until it lands. +func starting(h *Holder) { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + written, deleted, err := Publish(h.Store) + if err == nil { + if len(written)+len(deleted) > 0 { + logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted) + } + break } - wait = time.Hour + logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait) + time.Sleep(wait) + } + if n, err := h.RetireNeeds(); err != nil { + logf("[%s] the retired needs are not all deleted: %v", Module, err) + } else if n > 0 { + logf("[%s] deleted %d key(s) of the retired needs", Module, n) + } + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index 4661b226..6d48aadb 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -106,9 +106,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) { for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect, "shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect, "contribution": "mesh-provisioner_" + Collect, - "dir": "mesh-controller_assignments", "access": "mesh-controller_assignments", - "port": "mesh-controller_assignments", "seat": "mesh-controller_assignments", - "machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} { + "dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans", + "port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans", + "machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} { i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind }) if TheRegistry[i].Bucket != bucket { t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket) @@ -288,6 +288,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { Serves []string `json:"serves"` } `json:"claims"` State []string `json:"state"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` OwnSecrets map[string]string `json:"own-secrets"` } if err := json.Unmarshal(raw, &m); err != nil { @@ -300,7 +302,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { !slices.Equal(m.Claims[0].Serves, []string{Verb}) { t.Errorf("the claim: %+v", m.Claims) } - if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) { + // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } + // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools + // its own and the composition it does on a change. + served := map[string]bool{} + for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) { + served[tool.Name] = true + } + for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb, + Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool, + "a composition on a change": ComposeTool, "a decision on an answer": DecideTool} { + if !served[tool] { + t.Errorf("%s has no tool: %s is not served", what, tool) + } + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which is not served", tool) + } + } + for _, r := range []string{PlacementRead, WantsRead, ModulesRead, ManifestsRead, BuildsRead, HaveRead, ResolvedRead} { + if !slices.Contains(m.Reads, r) { + t.Errorf("the manifest does not read %s, which the composition reads", r) + } + } + for _, p := range Providers { + if !slices.Contains(m.Reads, boundRead(p)) { + t.Errorf("the manifest does not read %s's entries", p) + } + } + if slices.Contains(m.Reads, "mesh-controller.assignments") { + t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)") + } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside new file mode 100644 index 00000000..203aee31 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-provisioner/module.json diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go index 7d7d6f3e..15f0bb3f 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go @@ -48,16 +48,15 @@ type Answer struct { Decision string `json:"decision"` } -// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's -// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which -// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in -// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration, -// which the node-engine keeps for root alone. +// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR +// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state +// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for +// root alone, nor the controller's assignments. func placement(kind, what string) Answer { - return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments", + return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState, Key: ".", - Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." + - "placement)", Decision: "novox/hq ADR 0349 clause 6"} + Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " + + "(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"} } // oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1). diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index 9f0b6d60..70df969f 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -2,4 +2,4 @@ module mesh-provisioner go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index bb6beaa1..c728e39b 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 650425b3..1aba09af 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -11,11 +11,36 @@ ] } ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-assigner.placement", + "mesh-delivery-assigner.wants", + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.manifests", + "mesh-delivery-builder.builds", + "node-engine.have", + "mesh-config.resolved", + "gitea.bound", + "keycloak.bound", + "postgres.bound", + "route-adapter.bound", + "mesh-vault.bound" + ], + "tools": [ + "provisioner_provisions", + "provisioner_plans", + "provisioner_compose", + "provisioner_decide" + ], "state": [ "registry", "needs", "applied", - "collect" + "collect", + "provisions", + "plans" ], "own-secrets": { "broker": "${dir:state}/broker" From 6bee0b67896ac97523aa0fc9bcac6a14eb4db9d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:43:16 +0100 Subject: [PATCH 19/20] Never set a provision provided without its provider's outcome, read from the entry's state and why, and pin mesh-sdk 692dbf9 (issue 1954) A provider that writes no outcome leaves its keys asked (ADR 0570 clause 3), so a provider that fails is never hidden. --- checks/health/go.mod | 2 +- checks/health/go.sum | 4 +-- modules/mesh-config/go.mod | 2 +- modules/mesh-config/go.sum | 4 +-- modules/mesh-hand-acts/go.mod | 2 +- modules/mesh-hand-acts/go.sum | 4 +-- modules/mesh-healers/go.mod | 2 +- modules/mesh-healers/go.sum | 4 +-- .../cmd/mesh-provisioner/holder.go | 35 ++++++++++++++----- .../cmd/mesh-provisioner/holder_test.go | 10 +++--- modules/mesh-provisioner/go.mod | 2 +- modules/mesh-provisioner/go.sum | 4 +-- modules/mesh-self-check/go.mod | 2 +- modules/mesh-self-check/go.sum | 4 +-- 14 files changed, 51 insertions(+), 30 deletions(-) diff --git a/checks/health/go.mod b/checks/health/go.mod index 13b76586..8c3f521e 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 diff --git a/checks/health/go.sum b/checks/health/go.sum index 383c583f..387b26bd 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index fd98e5ad..e6d2dae0 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,7 +2,7 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 require ( github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 5ed89480..b8d81e43 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod index 6727bf7c..df0875c3 100644 --- a/modules/mesh-hand-acts/go.mod +++ b/modules/mesh-hand-acts/go.mod @@ -3,7 +3,7 @@ module mesh-hand-acts go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum index 326dffd4..a0208db4 100644 --- a/modules/mesh-hand-acts/go.sum +++ b/modules/mesh-hand-acts/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index e5cbdf2d..6351b3a4 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index 326dffd4..a0208db4 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index 04c2fb3e..b1382a79 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -12,8 +12,7 @@ package main // - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that // consumer, `_bound` under the same key: present is provided (the vault's generation carried into the // reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by -// itself, but for a provider that keeps no entries at all (its manifest declares no `bound`), which makes nothing per -// consumer. +// itself: a key of a provider that writes no outcome stays asked. // - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's // name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it; // a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state. @@ -292,6 +291,9 @@ func same(a, b provisions.Provision) bool { // entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what // the holder needs and nothing more. type entry struct { + // State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits. + State string `json:"state,omitempty"` + Why string `json:"why,omitempty"` Refused string `json:"refused,omitempty"` Waits string `json:"waits,omitempty"` VaultName string `json:"vault-name,omitempty"` @@ -299,12 +301,12 @@ type entry struct { } // outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has -// none. A provider that keeps no entries makes nothing per consumer: provided, said so. +// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key +// asked, said so. func (h *Holder) outcome(key string, p *provisions.Provision) { p.Ask, p.Warrant = "", nil if !keepsEntries(p.Provider.Module) { - p.State, p.Why = provisions.Provided, p.Provider.Module+" makes nothing per consumer: what it provides is its "+ - "placement and served values" + p.State, p.Why = provisions.Asked, p.Provider.Module+" writes no outcome yet: provided once it says it in its own entry" return } e, err := h.Store.Get(boundRead(p.Provider.Module), key) @@ -322,10 +324,17 @@ func (h *Holder) outcome(key string, p *provisions.Provision) { return } switch { - case en.Refused != "": + case en.State == provisions.Failed: + p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed") + case en.State == provisions.Waits: + p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person") + case en.State == "" && en.Refused != "": p.State, p.Why = provisions.Failed, en.Refused - case en.Waits != "": + case en.State == "" && en.Waits != "": p.State, p.Why = provisions.Waits, en.Waits + case en.State != "" && en.State != provisions.Provided: + p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is" + return default: p.State, p.Why = provisions.Provided, "" if en.VaultName != "" || en.Generation > 0 { @@ -349,7 +358,7 @@ func (h *Holder) retired(key string, p provisions.Provision) (bool, string) { return false, "" } if !keepsEntries(p.Provider.Module) { - return true, "its provider keeps nothing for it" + return false, "" } e, err := h.Store.Get(boundRead(p.Provider.Module), key) if err != nil || e != nil { @@ -615,3 +624,13 @@ func (h *Holder) RetireNeeds() (int, error) { } return n, nil } + +// orSaid is the first of the words that says something. +func orSaid(words ...string) string { + for _, w := range words { + if strings.TrimSpace(w) != "" { + return w + } + } + return "" +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go index 1164f728..398a338f 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -108,7 +108,7 @@ func (m *mesh) compose(t *testing.T, node string) Composed { } // A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry -// alone; a provider that keeps no entries makes nothing per consumer; the vault's generation reaches the reference; a +// alone; the vault's generation reaches the reference; a // quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2). func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { m := newMesh() @@ -124,7 +124,8 @@ func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { {"composed: asked", "", provisions.Asked, ""}, {"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"}, {"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"}, - {"the provider made it", `{"vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""}, + {"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"}, + {"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""}, } for _, s := range steps { if s.entry != "" { @@ -139,8 +140,9 @@ func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 { t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference) } - if p := m.state(t, "g14.gitea.split-dns"); p.State != provisions.Provided || !strings.Contains(p.Why, "makes nothing") { - t.Errorf("a provider keeping no entries: %+v", p) + // The holder never sets provided by itself: a provider writing no outcome leaves its key asked (clause 3). + if p := m.state(t, "g14.gitea.split-dns"); p.State != provisions.Asked || !strings.Contains(p.Why, "writes no outcome") { + t.Errorf("a provider writing no outcome: %+v", p) } if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { t.Error("the plan is not written") diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index 70df969f..b380be43 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -2,4 +2,4 @@ module mesh-provisioner go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index c728e39b..d2e21659 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod index 66a0e0d0..fd63c2a8 100644 --- a/modules/mesh-self-check/go.mod +++ b/modules/mesh-self-check/go.mod @@ -3,7 +3,7 @@ module mesh-self-check go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum index 326dffd4..a0208db4 100644 --- a/modules/mesh-self-check/go.sum +++ b/modules/mesh-self-check/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791 h1:HVKR7kENzltQVkfBN+2zhU9JOPZme7MjDkTvr/oiJ24= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029214150-692dbf9a1791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= From bc2e8b1fd246b8572095d8b0a709b716164c8574 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:44:59 +0100 Subject: [PATCH 20/20] Read the outcome of a provider that makes no credential per consumer from its own have, so its consumers' keys turn provided once it runs healthy at its wanted build (issue 1954) Failed is failed with the have's words; anything else stays asked, saying what it waits for. Providers that make a credential per consumer keep their bound entries. --- .../cmd/mesh-provisioner/holder.go | 39 ++++++++++++++++++- .../cmd/mesh-provisioner/holder_test.go | 23 +++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index b1382a79..a97e47c7 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -306,7 +306,7 @@ type entry struct { func (h *Holder) outcome(key string, p *provisions.Provision) { p.Ask, p.Warrant = "", nil if !keepsEntries(p.Provider.Module) { - p.State, p.Why = provisions.Asked, p.Provider.Module+" writes no outcome yet: provided once it says it in its own entry" + h.haveOutcome(p) return } e, err := h.Store.Get(boundRead(p.Provider.Module), key) @@ -343,6 +343,43 @@ func (h *Holder) outcome(key string, p *provisions.Provision) { } } +// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's +// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is +// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run. +func (h *Holder) haveOutcome(p *provisions.Provision) { + where := p.Provider.Module + " on " + p.Provider.Node + key := p.Provider.Node + "." + p.Provider.Module + e, err := h.Store.Get(HaveRead, key) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet" + return + } + have, err := delivery.DecodeHave(e.Value) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable" + return + } + wantFP := "" + if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil { + if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed { + wantFP = w.FP + } + } + switch { + case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed: + p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed") + case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP): + p.State, p.Why = provisions.Provided, "" + default: + p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+ + "health %s)", where, have.Phase, have.Health) + } +} + func keepsEntries(module string) bool { for _, m := range Providers { if m == module { diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go index 398a338f..4c319ee1 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -140,9 +140,26 @@ func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 { t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference) } - // The holder never sets provided by itself: a provider writing no outcome leaves its key asked (clause 3). - if p := m.state(t, "g14.gitea.split-dns"); p.State != provisions.Asked || !strings.Contains(p.Why, "writes no outcome") { - t.Errorf("a provider writing no outcome: %+v", p) + // A provider making no credential per consumer reports through its own have (clause 3): never provided by the + // holder alone. + split := "g14.gitea.split-dns" + for _, row := range []struct{ have, want, state, why string }{ + {"", "", provisions.Asked, "says nothing yet"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""}, + } { + if row.have != "" { + m.s.write(t, HaveRead, "g14.systemd-resolved", row.have) + } + if row.want != "" { + m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want) + } + m.compose(t, "g14") + if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) { + t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state) + } } if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { t.Error("the plan is not written")