From 464206d2debdcafb423f674f658d0fa4cafc80a4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:48:30 +0100 Subject: [PATCH 1/8] WM1 (step half): a head behind a pending revision of its branch raises no pipeline-stands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchman keeps the watched branch each open take-in wait names (take-in.pending's branch) and times a head only while its branch has none; once the revision is taken in the head is timed again from its own move. The head on a branch whose branch-unprotected stands (§497's second sentence) is not built: nothing the watchman hears says it (the forge bridge writes that finding, no event). Builds: design 47 G §12, §75, §497 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/watch/pipelines.go | 36 ++++++++++++++--- .../internal/watch/pipelines_test.go | 40 +++++++++++++++++++ .../internal/watch/takein.go | 28 +++++++++++++ .../internal/watch/watch.go | 4 +- 4 files changed, 101 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines.go b/modules/mesh-delivery-watchman/internal/watch/pipelines.go index bdec3fd6..09324691 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines.go @@ -24,7 +24,10 @@ const HeadBound = 5 * time.Minute const headRow = "head-pipeline" // head is a watched branch's head still waiting for its pipeline. -type head struct{ revision int64 } +type head struct { + revision int64 + at time.Time +} // SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline // is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5), @@ -48,15 +51,26 @@ func (w *Watchman) SourceMoved(body []byte) { if m.At != nil && !m.At.IsZero() { at = *m.At } - w.heads[m.Branch] = head{revision: m.Revision} - f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: at, - Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", m.Branch, m.Revision, stamp(at)), + w.heads[m.Branch] = head{revision: m.Revision, at: at} + w.timeHead(m.Branch) +} + +// timeHead times the head of a branch that waits for its pipeline, w.mu held: five minutes from its own time, unless the +// branch has a pending revision, whose take-in wait governs it (design 47 G §497) and which times it again once taken in. +func (w *Watchman) timeHead(branch string) { + h, ok := w.heads[branch] + if !ok || w.branchPending(branch) { + return + } + id := headID(branch) + f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: h.at, + Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", branch, h.revision, stamp(h.at)), Said: fmt.Sprintf("no pipeline in %s", HeadBound), Headline: "A change has started no pipeline", - Explanation: "The head of the watched branch " + m.Branch + " (source revision " + fmt.Sprint(m.Revision) + ") moved " + + Explanation: "The head of the watched branch " + branch + " (source revision " + fmt.Sprint(h.revision) + ") moved " + HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.", Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"} - left := HeadBound - w.now().Sub(at) + left := HeadBound - w.now().Sub(h.at) if left > 0 { w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f}) return @@ -64,6 +78,16 @@ func (w *Watchman) SourceMoved(body []byte) { w.find(f) } +// branchPending is whether an open take-in wait names the branch (design 47 G §497): its head is behind a pending revision. +func (w *Watchman) branchPending(branch string) bool { + for _, b := range w.pendingBranch { + if b == branch { + return true + } + } + return false +} + // PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that // source revision or a later one). A run on request is no head's pipeline. func (w *Watchman) PipelineStarted(body []byte) { diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index aa534388..f44ff602 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -7,6 +7,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) func moved(branch string, rev int64, group string, at time.Time) []byte { @@ -14,6 +15,21 @@ func moved(branch string, rev int64, group string, at time.Time) []byte { return b } +// takeInBodyOn is a take-in event's body that names its watched branch (design 47 G §493). +func takeInBodyOn(t *testing.T, kind, key, branch string, since time.Time) []byte { + t.Helper() + var m map[string]any + if err := json.Unmarshal(takeInBody(t, kind, key, since), &m); err != nil { + t.Fatal(err) + } + m["branch"] = branch + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return b +} + func started(branch string, rev int64, run string) []byte { b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"}) return b @@ -58,6 +74,30 @@ func TestAHeadWithNoPipelineStands(t *testing.T) { t.Fatalf("a pulse for a member: %v", o.pulses) } }, false}, + {"a head behind a pending revision of its branch is not timed", func(w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t)) + w.SourceMoved(moved("main", 7, "", c.t)) + if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { + t.Fatalf("a pulse for a head behind a pending revision: %v", o.pulses) + } + }, false}, + {"a pending revision of another branch does not hold it", func(w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "other", c.t)) + w.SourceMoved(moved("main", 7, "", c.t)) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + {"a head whose pending revision is taken in is timed again", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t)) + if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { + t.Fatalf("the head is timed behind a pending revision: %v", o.pulses) + } + c.t = c.t.Add(HeadBound) + w.TakeIn(takeInEvent(t, takein.TakeInTaken, "m.1"), takeInBodyOn(t, takein.TakeInTaken, "m.1", "main", c.t)) + if _, ok := o.findings[key]; !ok { + t.Fatalf("the head, past its five minutes when taken in, stands: %v", o.findings) + } + }, true}, {"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) { w.SourceMoved(moved("main", 7, "", c.t)) w.PipelineStarted(started("main", 7, "run-1")) diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index cc6b6c2e..4482b385 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -1,12 +1,14 @@ package watch import ( + "encoding/json" "fmt" "maps" "slices" "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) @@ -42,10 +44,14 @@ func (w *Watchman) TakeIn(event string, body []byte) { w.say("the body of %s says the kind %q, left", event, t.Kind) return } + // The wait names the watched branch it is of (design 47 G §493), by the contract's body. + var named meshcatalog.TakeInPending + _ = json.Unmarshal(body, &named) key := t.Key w.mu.Lock() defer w.mu.Unlock() w.takeIn(kind, key, t) + w.pendingOf(kind, key, named.Branch) } // SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every revision @@ -88,6 +94,28 @@ func (w *Watchman) SeedTakeIn(a takein.PendingAnswer) { } } +// pendingOf records or ends the branch an open take-in wait names (design 47 G §497), w.mu held: the head of a branch behind a +// pending revision raises no `pipeline-stands`, and is timed again, from the move of the head, once the branch has none. +func (w *Watchman) pendingOf(kind, key, branch string) { + if kind == takein.TakeInTaken { + branch = w.pendingBranch[key] + delete(w.pendingBranch, key) + if branch != "" && !w.branchPending(branch) { + w.timeHead(branch) + } + return + } + if branch == "" { + return + } + w.pendingBranch[key] = branch + if _, ok := w.heads[branch]; ok { + id := headID(branch) + w.unsignal(headRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) + } +} + // takeIn is one take-in event of key, w.mu held. func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { finding := FindingKey(ScopeDelivery, key, KindTakeInWaits) diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 78d77652..bfe049f7 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -118,6 +118,8 @@ type Watchman struct { awaited map[string]bool // heads is, by watched branch, the head still waiting for its pipeline (pipelines.go). heads map[string]head + // pendingBranch is, by key, the watched branch of each open take-in wait that names one (takein.go, design 47 G §497). + pendingBranch map[string]string // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). queued map[string]map[string]time.Time // awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go). @@ -130,7 +132,7 @@ type Watchman struct { func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound, - findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, away: map[string]bool{}, + findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, pendingBranch: map[string]string{}, away: map[string]bool{}, haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, From 62de8d0bb81ddd54f5578b0bb02ad7116788034c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:49:48 +0100 Subject: [PATCH 2/8] WM4, WM5: the watchman hears the node-engine's wait-held and wait-ended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hold of the composition standing past 30 minutes is delivery-waits (node, module, reason), a wait for a kept build past 60 minutes is kept-build-waits; each clears on wait-ended. The node-engine's version is not yet named in them (it needs the catalog entry of the fingerprint running.. names). Builds: design 47 G §499, §541, §271, §376 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-watchman/consumes.go | 20 ++++++ .../cmd/mesh-delivery-watchman/main_test.go | 14 ++++ .../internal/watch/waits.go | 71 +++++++++++++++++++ .../internal/watch/waits_test.go | 36 ++++++++++ .../mesh-delivery-watchman/manifest_test.go | 2 +- modules/mesh-delivery-watchman/module.json | 2 + 6 files changed, 144 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index 912f1fe4..864cab05 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -46,6 +47,7 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, } w.Accepted(a) case pipelineSubject(w, subject, body): + case engineWaitSubject(w, subject, body): case recordSubject(w, subject, body): case buildSeatSubject(w, subject, body): case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated): @@ -190,6 +192,24 @@ func recordSubject(w *watch.Watchman, subject string, body []byte) bool { return true } +// engineWaitSubject hands the node-engine seat's `wait-held` and `wait-ended`, on +// mesh.seat.node-engine.event..., to the watchman (design 47 G §499, §541). It says whether the subject was one. +func engineWaitSubject(w *watch.Watchman, subject string, body []byte) bool { + parts := strings.Split(subject, ".") + if len(parts) != 7 || parts[0] != "mesh" || parts[1] != "seat" || parts[2] != nodeengine.Seat || parts[3] != "event" { + return false + } + switch parts[4] { + case nodeengine.EventWaitHeld: + w.EngineWaitHeld(parts[5], parts[6], body) + case nodeengine.EventWaitEnded: + w.EngineWaitEnded(parts[5], parts[6]) + default: + return false + } + return true +} + // buildSeatSubject hands the build seats' `ask-taken`, `step-progress` and `step-ended`, on // mesh.seat..event.., to the watchman (design 47 G §483, §496, §602, A-136): it times every ask of a // build seat from them. It says whether the subject was one of them. diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index a0f47f2d..aecea500 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -75,6 +75,20 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Fatalf("created left %v", out.findings) } }) + t.Run("design 47 G §499, §541: a node-engine's wait-held and wait-ended reach the watchman by their subjects", func(t *testing.T) { + out := &findingsOut{findings: map[string]bool{}} + w := watch.New(out, time.Now) + say := func(string, ...any) {} + body := fmt.Sprintf(`{"step":"compose","reason":"entry-missing","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339)) + consumed(w, "mesh.seat.node-engine.event.wait-held.ace.web", []byte(body), say) + if !out.findings[watch.FindingKey(watch.ScopeDelivery, "ace.web", watch.KindDeliveryWaits)] { + t.Fatalf("a hold two hours old reached %v", out.findings) + } + consumed(w, "mesh.seat.node-engine.event.wait-ended.ace.web", []byte(body), say) + if len(out.findings) != 0 { + t.Fatalf("wait-ended left %v", out.findings) + } + }) err := run(context.Background(), func(string) string { return "" }, io.Discard) if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err) diff --git a/modules/mesh-delivery-watchman/internal/watch/waits.go b/modules/mesh-delivery-watchman/internal/watch/waits.go index b9d4d0c4..c9931cd1 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) @@ -110,3 +111,73 @@ func (w *Watchman) putStands(id string, a meshwants.Wait, bound time.Duration, f } w.find(f) } + +// What a node-engine says of a hold it keeps (design 47 G §499, §541), heard on the seat node-engine's `wait-held` and `wait-ended` +// (`mesh.seat.node-engine.event...`, the node the one that says it): a hold of the composition (`step: +// compose`, with its reason) standing past composing's 30 minutes (§266) is the finding `delivery-waits`, naming the node, the +// module and the reason, and a wait for a kept build (`step: kept-build`, naming the build node) standing past building's default +// bound of 60 minutes is `kept-build-waits`. Each clears on `wait-ended`. + +// The findings of a node-engine's waits, and the bounds that stand when its event names none. +const ( + KindDeliveryWaits = "delivery-waits" + KindKeptBuildWaits = "kept-build-waits" + ComposeBound = 30 * time.Minute + KeptBuildBound = 60 * time.Minute +) + +// engineRow is the row of the signals table of a node-engine's wait. +const engineRow = "engine-wait" + +func engineWaitID(node, module string) string { return node + "." + module } + +// EngineWaitHeld hears the node-engine of node saying `wait-held` for module. +func (w *Watchman) EngineWaitHeld(node, module string, body []byte) { + var a nodeengine.Wait + if err := json.Unmarshal(body, &a); err != nil || a.Since.IsZero() { + w.say("a wait-held of the node-engine of %s for %s does not read, left: %v", node, module, err) + return + } + kind, bound := KindDeliveryWaits, ComposeBound + if a.Step == nodeengine.WaitStepKeptBuild { + kind, bound = KindKeptBuildWaits, KeptBuildBound + } + if d, err := time.ParseDuration(a.Bound); err == nil && d > 0 { + bound = d + } + from := a.Since + if a.ClockFrom != nil { + from = *a.ClockFrom + } + id := engineWaitID(node, module) + what := "holds the composition of " + module + if a.Step == nodeengine.WaitStepKeptBuild { + what = "waits for the kept build of " + module + " on " + orWords(a.Node, "no node") + } + f := Finding{Scope: ScopeDelivery, ID: id, Kind: kind, Severity: Warning, Machine: node, Source: engineRow, Since: a.Since, + Summary: fmt.Sprintf("the node-engine of %s %s: %s, since %s (bound %s)", node, what, orWords(a.Reason, "no reason said"), stamp(a.Since), bound), + Said: "held for " + bound.String(), + Headline: "A module's delivery is held on a machine", + Explanation: "The node-engine of " + node + " " + what + " (" + orWords(a.Reason, "no reason said") + ") and its bound of " + + bound.String() + " has run, so the module is not at the build wanted there.", + Needs: "what it waits on to come: " + orWords(a.Words, orWords(a.Reason, "the node-engine's wait to end"))} + w.mu.Lock() + defer w.mu.Unlock() + w.unsignal(engineRow, id) + w.unfind(f.Key()) + if left := bound - w.now().Sub(from); left > 0 { + w.signal(engineRow, id, left, signalDue{Do: dueFind, Finding: f}) + return + } + w.find(f) +} + +// EngineWaitEnded hears the node-engine of node saying `wait-ended` for module: the hold is over, whichever kind it was. +func (w *Watchman) EngineWaitEnded(node, module string) { + id := engineWaitID(node, module) + w.mu.Lock() + defer w.mu.Unlock() + w.unsignal(engineRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindDeliveryWaits)) + w.unfind(FindingKey(ScopeDelivery, id, KindKeptBuildWaits)) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/waits_test.go b/modules/mesh-delivery-watchman/internal/watch/waits_test.go index 481bd363..9e587b8c 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits_test.go @@ -2,9 +2,11 @@ package watch import ( "encoding/json" + "strings" "testing" "time" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) @@ -93,3 +95,37 @@ func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) { t.Fatalf("wait-ended left %v", o.findings) } } + +// design 47 G §499, §541 (WM4, WM5): a node-engine's hold of the composition standing 30 minutes is `delivery-waits` naming the node, +// the module and the reason, a wait for a kept build standing 60 minutes is `kept-build-waits`, and `wait-ended` clears either. +func TestANodeEnginesWaitIsAFindingPastItsBound(t *testing.T) { + held := func(step, reason string, since time.Time) []byte { + b, _ := json.Marshal(nodeengine.Wait{Step: step, Reason: reason, Node: "build1", Since: since, Owner: "node-engine"}) + return b + } + for _, tc := range []struct { + name, step, reason, kind string + bound time.Duration + }{ + {"a hold of the composition", nodeengine.WaitStepCompose, "restore point missing", KindDeliveryWaits, 30 * time.Minute}, + {"a wait for a kept build", nodeengine.WaitStepKeptBuild, "node away", KindKeptBuildWaits, 60 * time.Minute}, + } { + t.Run(tc.name, func(t *testing.T) { + w, o, c := setup() + id := "ace.web" + w.EngineWaitHeld("ace", "web", held(tc.step, tc.reason, c.t)) + if o.ttls[signalKey(engineRow, id)] != tc.bound || len(o.findings) != 0 { + t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(engineRow, id)], o.findings) + } + expire(w, c, signalKey(engineRow, id), tc.bound) + f, ok := o.findings[FindingKey(ScopeDelivery, id, tc.kind)] + if !ok || len(o.findings) != 1 || !strings.Contains(f.Summary, tc.reason) { + t.Fatalf("past its bound: %v", o.findings) + } + w.EngineWaitEnded("ace", "web") + if len(o.findings) != 0 { + t.Fatalf("wait-ended left %v", o.findings) + } + }) + } +} diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 8fb2ae9a..dea3bc92 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -130,7 +130,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 52e7103b..5e6fe470 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -67,6 +67,8 @@ "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", + "node-engine.wait-held", + "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked" From 524fbb02ef66d83759aa1936d7e3a39cbb9ce8f2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:52:10 +0100 Subject: [PATCH 3/8] WM13 (builder half): a key queued behind a package says wait-held and wait-ended in the one wait form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder says wait-held for a key queued behind a package, the package as its part and on (package:), on its stream waits, and wait-ended published once the key is claimed; a toolchain wait never ends by that, and a package wait never by a toolchain's end. Files: internal/build/waits.go (Package, On, WaitPublished), internal/build/run.go (Hold at the queued result, End at the claim), cmd/mesh-delivery-builder/waits.go (Part in the body, the End rule). Builds: design 47 G A-192, §499 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-builder/waits.go | 16 ++++++++++---- .../cmd/mesh-delivery-builder/waits_test.go | 21 +++++++++++++++++++ .../internal/build/r4e_test.go | 17 +++++++++++++++ .../internal/build/run.go | 7 +++++++ .../internal/build/waits.go | 16 +++++++++++--- .../internal/build/waits_test.go | 3 ++- 6 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go index 29005677..e3781b5d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go @@ -18,8 +18,10 @@ import ( // waitBody is the body of wait-held and wait-ended on the builder's stream waits (design 47 G §672): the fields of §629's // wait with `step` building, `on` toolchain:@, `module` and `fingerprint`, and, on wait-ended, `how`. type waitBody struct { - Step string `json:"step"` - On string `json:"on"` + Step string `json:"step"` + On string `json:"on"` + // Part is the package a key queued behind it waits for (design 47 G A-192); none for a toolchain wait. + Part string `json:"part,omitempty"` Module string `json:"module"` Fingerprint string `json:"fingerprint"` Since time.Time `json:"since"` @@ -46,7 +48,7 @@ func newWaitsKept(kv *process.AwaitedBucket, say func(context.Context, string, s } func (w *waitsKept) body(t build.ToolchainWait, how string) waitBody { - return waitBody{Step: waitStepBuilding, On: t.On(), Module: t.Module, Fingerprint: t.Fingerprint, Since: t.Since, Bound: w.bound.String(), How: how} + return waitBody{Step: waitStepBuilding, On: t.On(), Part: t.Package, Module: t.Module, Fingerprint: t.Fingerprint, Since: t.Since, Bound: w.bound.String(), How: how} } // start reads the state whole and says wait-held again for each key. @@ -110,7 +112,13 @@ func (w *waitsKept) End(ctx context.Context, module, fp, how string) bool { } w.mu.Lock() t, had := w.held[key] - delete(w.held, key) + // A key queued behind a package ends only when it is published, and a toolchain wait never by that (design 47 G A-192). + if had && (t.Package != "") != (how == build.WaitPublished) { + had = false + } + if had { + delete(w.held, key) + } w.mu.Unlock() if !had { return false diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits_test.go new file mode 100644 index 00000000..b21a290f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "testing" + "time" + + "mesh-delivery-builder/internal/build" +) + +// Design 47 G A-192: a key queued behind a package says its wait in the one wait form, the package as its `part` and `on`; a toolchain +// wait names none. +func TestAWaitBodyNamesItsPart(t *testing.T) { + w := &waitsKept{bound: time.Hour, held: map[string]build.ToolchainWait{}} + pkg := w.body(build.ToolchainWait{Module: "m", Fingerprint: "src2:f", Package: "@novox/sdk@0.1.15"}, "") + if pkg.Part != "@novox/sdk@0.1.15" || pkg.On != "package:@novox/sdk@0.1.15" || pkg.Step != waitStepBuilding { + t.Errorf("a package wait says %+v", pkg) + } + if tc := w.body(build.ToolchainWait{Module: "m", Fingerprint: "src2:f", Language: "go", Version: "go1.26.0"}, ""); tc.Part != "" || tc.On != "toolchain:go@go1.26.0" { + t.Errorf("a toolchain wait says %+v", tc) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index c1957e8f..a79c0ed0 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -173,6 +173,8 @@ func sdkModule(t *testing.T, r *rig, ref string) []byte { func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePublished(t *testing.T) { // B7, ADR 0500 clause 6 const ref = "@novox/mesh-sdk@0.1.15" r := newRig(t, tsBundle) + w := &memWaits{} + r.b.Waits = w for _, n := range []string{"ace", "novox"} { r.bm.fails[n+"/build_node_fetch"] = waitsFor + ref + ": the lock names a version the registry does not hold" } @@ -180,6 +182,15 @@ func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePubli if v.State != builds.BuildQueued || v.Waits != ref || v.Error != "" { t.Fatalf("build = %+v, want queued, waiting for %s, not failed", v, ref) } + // Design 47 G A-192: the queued key is a wait said in the one wait form, with its package as the part, and it stands while the package is unpublished. + if len(w.held) != 1 { + t.Fatalf("waits held %v, want the one of the package", w.held) + } + for _, h := range w.held { + if h.Package != ref || h.On() != "package:"+ref { + t.Errorf("the wait is %+v on %s, want the package %s", h, h.On(), ref) + } + } delete(r.bm.fails, "ace/build_node_fetch") delete(r.bm.fails, "novox/build_node_fetch") writes := r.st.writes @@ -188,11 +199,17 @@ func TestAModuleWhoseLockNamesAnUnpublishedPackageWaitsQueuedAndIsBuiltOncePubli if r.st.writes != writes { t.Fatalf("a key waiting for an unpublished package was claimed (%d writes, was %d)", r.st.writes, writes) } + if len(w.held) != 1 { + t.Fatalf("the wait ended while the package is unpublished: %v", w.held) + } r.b.Module(context.Background(), "sdk", sdkModule(t, r, ref)) // the package's trunk key is built and published r.b.Wait() if v = valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Waits != "" { t.Errorf("build = %+v, want built once the package it waits for is published", v) } + if len(w.held) != 0 || len(w.ended) != 1 || w.ended[0] != WaitPublished { + t.Errorf("held %v, ended %v: want the wait ended published", w.held, w.ended) + } } func TestAKeyWaitingForAPackageAlreadyPublishedIsBuiltWhenTheBuilderStarts(t *testing.T) { // B7: the scan at start diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index a6a59251..86ebbfe8 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -617,6 +617,9 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } b.handClaimed(key, cl.Rev) + if b.Waits != nil { + b.Waits.End(ctx, module, m.Fp2, WaitPublished) // a key queued behind a package is claimed once it is published + } b.logBuild(LogStarted(module, m, "", b.Node, b.now())) if cl.Unasked != nil { b.logf("%v", cl.Unasked) @@ -717,6 +720,10 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } if result.State == builds.BuildQueued { b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits) + if b.Waits != nil { + // Said as a wait in the one wait form, with its part, for the watchman to time (design 47 G A-192; ADR 0603 §8). + b.Waits.Hold(ctx, ToolchainWait{Module: module, Fingerprint: m.Fp2, Package: result.Waits, Since: b.now()}) + } } if b.notePublished(result) { // A key may wait for the package this build published (ADR 0500 clause 6). diff --git a/modules/mesh-delivery-builder/internal/build/waits.go b/modules/mesh-delivery-builder/internal/build/waits.go index b0f6d225..d314976c 100644 --- a/modules/mesh-delivery-builder/internal/build/waits.go +++ b/modules/mesh-delivery-builder/internal/build/waits.go @@ -26,8 +26,10 @@ const ( // ModeRelease is the Mode of a reasks value of builder_release. ModeRelease = "release" // WaitRan ends a wait when a toolchain-agreed names the version; WaitReleased when builder_release ended it. - WaitRan = "ran" - WaitReleased = "released" + WaitRan = "ran" + // WaitPublished ends the wait of a key queued behind a package, when the key is claimed (the package is published). + WaitPublished = "published" + WaitReleased = "released" ) // ToolchainWait is one wait: the entry's module and fingerprint, and the toolchain it waits for. @@ -37,10 +39,18 @@ type ToolchainWait struct { Language string `json:"language"` Version string `json:"version"` Since time.Time `json:"since"` + // Package is the package version a key queued behind it waits for, `@` (ADR 0603 §8; design 47 G A-192): the wait is the + // package's, not a toolchain's, and Language and Version are empty. + Package string `json:"package,omitempty"` } // On is what the wait waits for, as the wait's `on` says it. -func (w ToolchainWait) On() string { return "toolchain:" + w.Language + "@" + w.Version } +func (w ToolchainWait) On() string { + if w.Package != "" { + return "package:" + w.Package + } + return "toolchain:" + w.Language + "@" + w.Version +} // Waits keeps the builder's waits: Hold writes one in the state waits and says wait-held, once per wait; End deletes it and // says wait-ended with how, and answers whether one was held. diff --git a/modules/mesh-delivery-builder/internal/build/waits_test.go b/modules/mesh-delivery-builder/internal/build/waits_test.go index 2a63722f..72eb4608 100644 --- a/modules/mesh-delivery-builder/internal/build/waits_test.go +++ b/modules/mesh-delivery-builder/internal/build/waits_test.go @@ -28,7 +28,8 @@ func (w *memWaits) End(_ context.Context, module, fp, how string) bool { w.mu.Lock() defer w.mu.Unlock() k := module + "." + fp - if _, ok := w.held[k]; !ok { + t, ok := w.held[k] + if !ok || (t.Package != "") != (how == WaitPublished) { // as the builder's waitsKept ends them: a package wait by its publishing alone return false } delete(w.held, k) From faec6b53338a94b09d809e8543cba9f570a38c80 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:53:23 +0100 Subject: [PATCH 4/8] WM13 (watchman half): the builder's builds are no longer read; a package wait is heard as wait-held MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchman's read of mesh-delivery-builder.builds (Build in mirror.go, the bucket, its manifest reads) goes; it hears the builder's wait-held and wait-ended on the builder's stream, naming a package as their part, and times them as an acting part's wait. A toolchain wait (no part) is left to the builder's own finding. The node-engine's have and mesh-host.health reads stay: no event carries a have's phase, kept data, node record or a node's health report (asked of the lead). Builds: design 47 G A-192, §484, §496, §602, §548 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-watchman/consumes.go | 20 ++++++++++ .../cmd/mesh-delivery-watchman/main.go | 16 -------- .../cmd/mesh-delivery-watchman/main_test.go | 13 +++++++ .../internal/watch/findings_test.go | 13 ++----- .../internal/watch/mirror.go | 39 ------------------- .../internal/watch/names.go | 3 -- .../internal/watch/waits.go | 26 +++++++++++++ .../internal/watch/waits_test.go | 29 ++++++++++++++ .../mesh-delivery-watchman/manifest_test.go | 5 +-- modules/mesh-delivery-watchman/module.json | 3 +- 10 files changed, 95 insertions(+), 72 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index 864cab05..d176cf4f 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -48,6 +48,7 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, w.Accepted(a) case pipelineSubject(w, subject, body): case engineWaitSubject(w, subject, body): + case moduleWaitSubject(w, subject, body): case recordSubject(w, subject, body): case buildSeatSubject(w, subject, body): case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated): @@ -210,6 +211,25 @@ func engineWaitSubject(w *watch.Watchman, subject string, body []byte) bool { return true } +// moduleWaitSubject hands a module's `wait-held` and `wait-ended` on its own stream, on +// mesh.stream..event..., to the watchman: the form the builder says a key queued behind a package +// in (design 47 G A-192). It says whether the subject was one. +func moduleWaitSubject(w *watch.Watchman, subject string, body []byte) bool { + parts := strings.Split(subject, ".") + if len(parts) != 7 || parts[0] != "mesh" || parts[1] != "stream" || parts[3] != "event" { + return false + } + switch parts[4] { + case nodeengine.EventWaitHeld: + w.ModuleWaitHeld(parts[2], body) + case nodeengine.EventWaitEnded: + w.ModuleWaitEnded(parts[2], body) + default: + return false + } + return true +} + // buildSeatSubject hands the build seats' `ask-taken`, `step-progress` and `step-ended`, on // mesh.seat..event.., to the watchman (design 47 G §483, §496, §602, A-136): it times every ask of a // build seat from them. It says whether the subject was one of them. diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index f3332391..93b5454c 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -323,17 +323,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro haveSeen = nil } } - // The builder's builds (ADR 0603 §8): a key queued behind a package is a wait, mirrored from the time it was written. - var buildSeen jetstream.KeyWatcher - if kv := process.OptionalBucket(ctx, js, watch.BuilderBuildsBucket, "a build waiting for a package is not watched until the watchman starts again", logger.Printf); kv != nil { - buildEach := func(e jetstream.KeyValueEntry) { w.Build(e.Key(), e.Value(), opOf(e), e.Created()) } - if buildSeen, err = settle(ctx, kv, buildEach); err != nil { - process.NotWatched(logger.Printf, watch.BuilderBuildsBucket, err, "a build waiting for a package is not watched until the watchman starts again") - buildSeen = nil - } else { - w.ReadWhole(watch.BuilderBuildsBucket) - } - } // The events first, then the one request of the open connections, so no connection falls between them. onEvent := func(open bool) nats.MsgHandler { return func(m *nats.Msg) { @@ -491,11 +480,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() - if buildSeen != nil { - go func() { - errs <- follow(ctx, buildSeen, func(e jetstream.KeyValueEntry) { w.Build(e.Key(), e.Value(), opOf(e), e.Created()) }) - }() - } go func() { for { select { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index aecea500..1c662686 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -89,6 +89,19 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Fatalf("wait-ended left %v", out.findings) } }) + t.Run("design 47 G A-192: a module's wait-held on its stream, naming a part, reaches the watchman", func(t *testing.T) { + out := &findingsOut{findings: map[string]bool{}} + w := watch.New(out, time.Now) + body := fmt.Sprintf(`{"step":"building","part":"mesh-sdk@v0.1.14","module":"web","fingerprint":"src2:f","bound":"1h0m0s","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339)) + consumed(w, "mesh.stream.mesh-delivery-builder.event.wait-held.web.src2:f", []byte(body), func(string, ...any) {}) + if len(out.findings) != 1 { + t.Fatalf("a package wait two hours old reached %v", out.findings) + } + consumed(w, "mesh.stream.mesh-delivery-builder.event.wait-ended.web.src2:f", []byte(body), func(string, ...any) {}) + if len(out.findings) != 0 { + t.Fatalf("wait-ended left %v", out.findings) + } + }) err := run(context.Background(), func(string) string { return "" }, io.Discard) if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { t.Fatalf("run = %v, want a refusal naming MESH_BROKER_FILE", err) diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 521f8557..6214cee1 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -12,7 +12,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -737,7 +736,7 @@ func TestTheWatchmansFindings(t *testing.T) { w.DeadLettersHeld(map[string]int{"SEAT_X.y_worker": 1}, who) w.NoticesWaiting(true) }, map[string]string{"bus.SEAT_X.y_worker.max-deliveries": Warning, "bus.DEAD_LETTER_NOTICES.controller.notices-waiting": Warning}}, - {"ADR 0603 §8: a want held by a rule, a provision asked and a build queued behind a package each raise a finding naming the wait past their bound, and clear when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"ADR 0603 §8: a want held by a rule and a provision asked each raise a finding naming the wait past their bound, and clear when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.SetStepBound(30 * time.Minute) held, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it yet"}) released, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Self: true}) @@ -745,15 +744,11 @@ func TestTheWatchmansFindings(t *testing.T) { Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, At: t0}) provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Provided, At: t0}) - bkey, _ := builds.BuilderBuildKey("web", fpNew) - queued, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildQueued, Waits: "mesh-sdk@v0.1.14"}) - building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "n1"}) w.Have("ace.web", haveValue(t, fpOld), Put) // the node runs what the held want names: the hold is still the wait w.Want("want.ace.web", held, Put, t0) w.Provision("ace.photos.oidc", asked, Put) - w.Build(bkey, queued, Put, t0) - waits := map[string]string{AssignerWantsBucket: "want.ace.web", ProvisionsBucket: "ace.photos.oidc", BuilderBuildsBucket: bkey} - if len(o.pulses) != 3 || len(o.findings) != 0 { + waits := map[string]string{AssignerWantsBucket: "want.ace.web", ProvisionsBucket: "ace.photos.oidc"} + if len(o.pulses) != 2 || len(o.findings) != 0 { t.Fatalf("pulses %v findings %v", o.pulses, o.findings) } // The keys are written again as their parts move on, with new words and a new time: the wait still began at t0. @@ -763,7 +758,6 @@ func TestTheWatchmansFindings(t *testing.T) { Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, Why: "the provider is applying", At: c.t}) w.Want("want.ace.web", held2, Put, c.t) w.Provision("ace.photos.oidc", asked2, Put) - w.Build(bkey, queued, Put, c.t) for bucket, key := range waits { expire(w, c, pulseKeyOf(t, bucket, key), 31*time.Minute) } @@ -775,7 +769,6 @@ func TestTheWatchmansFindings(t *testing.T) { } w.Want("want.ace.web", released, Put, c.t) w.Provision("ace.photos.oidc", provided, Put) - w.Build(bkey, building, Put, c.t) }, map[string]string{}}, } for _, tc := range cases { diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index e6914293..f20ceefb 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -8,7 +8,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" @@ -165,44 +164,6 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { describe("the node agent of %s to run %s at %s", node, module, shortFP(want.Fp)), want.Fp, at, 0) } -// A build key queued behind a package is a wait of the builder's (ADR 0603 §8, ADR 0500 clause 6): its Waits names the package -// version the registry does not yet hold, and the key is not claimed until it is published. Its pulse is the pulse of the -// builder's builds bucket under the key, from the time the key was written, bounded as a step is (StepBound); the key -// claimed, done, failed or deleted ends it. - -// Build is a change of a key of the builder's builds bucket, read from the watch (at the start the keys that stand, then -// each change), at the time the entry was written. -func (w *Watchman) Build(key string, value []byte, op Op, at time.Time) { - module, fp, err := builds.ParseBuilderBuildKey(key) - if err != nil { - return - } - w.mu.Lock() - defer w.mu.Unlock() - if op == Expired { - return - } - if op == Delete { - w.moved(BuilderBuildsBucket, key) - return - } - w.stands(BuilderBuildsBucket, key) - b, err := builds.DecodeBuilderBuild(value) - if err != nil { - w.say("the build %s cannot be read, and is left: %v", key, err) - return - } - if b.State != builds.BuildQueued || b.Waits == "" { - w.moved(BuilderBuildsBucket, key) - return - } - if at.IsZero() { - at = w.now() - } - w.mirrorKept(BuilderBuildsBucket, key, - describe("the package %s to be published before the build of %s at %s, bound %s", b.Waits, module, shortFP(fp), w.stepBound), b.Waits, at) -} - // Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the // fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left. func (w *Watchman) Have(key string, value []byte, op Op) { diff --git a/modules/mesh-delivery-watchman/internal/watch/names.go b/modules/mesh-delivery-watchman/internal/watch/names.go index c081279c..5759cc5b 100644 --- a/modules/mesh-delivery-watchman/internal/watch/names.go +++ b/modules/mesh-delivery-watchman/internal/watch/names.go @@ -14,7 +14,6 @@ const ( ProvisionsBucket = "mesh-provisioner_provisions" // IndexerModule is the holder of the seat mesh-catalog, named from this module's manifest (consumes and invokes of mesh-delivery-indexer). IndexerModule = "mesh-delivery-indexer" - builderModule = "mesh-delivery-builder" // The watchman's own states of the seat mesh-watchdogs. PresenceBucket = WatchmanModule + "_" + meshwatchdogs.StatePresence @@ -24,8 +23,6 @@ const ( // HaveBucket and NodeHealthBucket are the node-engine's states have and health (reads: node-engine.have, mesh-host.health). HaveBucket = "node-engine_have" NodeHealthBucket = "mesh-host_health" - // BuilderBuildsBucket is the builder's state builds (reads: mesh-delivery-builder.builds). - BuilderBuildsBucket = builderModule + "_builds" // AssignerWantsBucket is the assigner's state wants, followed by its views family (consumes: mesh-delivery-assigner.views.wants). AssignerWantsBucket = AssignerModule + "_wants" // ModulesBucket is the indexer's state modules, followed by its views family (consumes: mesh-delivery-indexer.views.modules). diff --git a/modules/mesh-delivery-watchman/internal/watch/waits.go b/modules/mesh-delivery-watchman/internal/watch/waits.go index c9931cd1..235304c9 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits.go @@ -181,3 +181,29 @@ func (w *Watchman) EngineWaitEnded(node, module string) { w.unfind(FindingKey(ScopeDelivery, id, KindDeliveryWaits)) w.unfind(FindingKey(ScopeDelivery, id, KindKeptBuildWaits)) } + +// ModuleWaitHeld hears a module's `wait-held` on its own stream (`mesh.stream..event.wait-held..`), the form +// the builder says a key queued behind a package in, with the package as its `part` (design 47 G A-192, ADR 0603 §8): it is timed as any +// acting part's wait. A wait naming no part is a toolchain wait, which the builder times itself (§672), and is left. +func (w *Watchman) ModuleWaitHeld(holder string, body []byte) { + var a meshwants.Wait + if json.Unmarshal(body, &a) == nil && a.Part == "" { + return + } + a2, ok := decodeWait(w, holder, "wait-held", body) + if !ok { + return + } + w.mu.Lock() + defer w.mu.Unlock() + w.waitHeld(holder, a2) +} + +// ModuleWaitEnded hears the same module's `wait-ended`. +func (w *Watchman) ModuleWaitEnded(holder string, body []byte) { + var a meshwants.Wait + if json.Unmarshal(body, &a) == nil && a.Part == "" { + return + } + w.WaitEnded(holder, body) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/waits_test.go b/modules/mesh-delivery-watchman/internal/watch/waits_test.go index 9e587b8c..b45b150b 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits_test.go @@ -129,3 +129,32 @@ func TestANodeEnginesWaitIsAFindingPastItsBound(t *testing.T) { }) } } + +// design 47 G A-192 (WM13): a key queued behind a package is a wait the builder says on its own stream with the package as its part; +// past its bound it is `acting-part-late`, `wait-ended` clears it, and a toolchain wait (no part) is left to the builder. +func TestAKeyQueuedBehindAPackageIsAnActingPartsWait(t *testing.T) { + body := func(part string, since time.Time) []byte { + b, _ := json.Marshal(meshwants.Wait{Step: "building", Part: part, On: "package:" + part, Module: "web", Fingerprint: "src2:f", Since: since, Bound: "1h0m0s"}) + return b + } + const holder = "builder" + w, o, c := setup() + id := waitID(holder, meshwants.Wait{Step: "building", Part: "mesh-sdk@v0.1.14", Module: "web"}) + w.ModuleWaitHeld(holder, body("mesh-sdk@v0.1.14", c.t)) + if o.ttls[signalKey(actingRow, id)] != time.Hour || len(o.findings) != 0 { + t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(actingRow, id)], o.findings) + } + expire(w, c, signalKey(actingRow, id), time.Hour) + if _, ok := o.findings[FindingKey(ScopeDelivery, id, KindActingPartLate)]; !ok || len(o.findings) != 1 { + t.Fatalf("past its bound: %v", o.findings) + } + w.ModuleWaitEnded(holder, body("mesh-sdk@v0.1.14", c.t)) + if len(o.findings) != 0 { + t.Fatalf("wait-ended left %v", o.findings) + } + w, o, c = setup() + w.ModuleWaitHeld(holder, body("", c.t.Add(-2*time.Hour))) + if len(o.findings) != 0 || len(o.ttls) != 0 { + t.Fatalf("a toolchain wait was timed by the watchman: %v %v", o.ttls, o.findings) + } +} diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index dea3bc92..27f85ec6 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -103,7 +103,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 reads := slices.Clone(m.Reads) slices.Sort(reads) // The build seats' steps are no read (design 47 G §483, A-136): the watchman times an ask from the seats' step-progress. - if !reflect.DeepEqual(reads, []string{"mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { + if !reflect.DeepEqual(reads, []string{"mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } // Design 47 G §368: the version is MAJOR.MINOR and nothing else. @@ -130,7 +130,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). @@ -174,7 +174,6 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 "stuck": {watch.StuckBucket, m.Module + "_" + m.State[2].Name}, "have": {watch.HaveBucket, bucket(m.Reads, "node-engine.have")}, "health": {watch.NodeHealthBucket, bucket(m.Reads, "mesh-host.health")}, - "builds": {watch.BuilderBuildsBucket, bucket(m.Reads, "mesh-delivery-builder.builds")}, "wants": {watch.AssignerWantsBucket, bucket(m.Consumes, "mesh-delivery-assigner.views.wants")}, "provisions": {watch.ProvisionsBucket, bucket(m.Reads, "mesh-provisioner.provisions")}, "modules": {watch.ModulesBucket, bucket(m.Consumes, "mesh-delivery-indexer.views.modules")}, diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 5e6fe470..d74809c9 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -16,7 +16,6 @@ } ], "reads": [ - "mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have" @@ -67,6 +66,8 @@ "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", + "mesh-delivery-builder.wait-held", + "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", From 1abc5ae6d92662e62b2e56f27375e320eea2ca1a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:55:02 +0100 Subject: [PATCH 5/8] WM9, WM15: the manifest findings are the indexer's verdicts heard as events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit manifest-refused raises manifest-refused; manifest-accepted, which carries problems, raises manifest-outgrown or clears it and any refusal; the start seeds both from indexer_verdicts. The indexer's modules view (ModuleVerdict, views.modules, indexer_read_modules) is no longer followed, and manifest-waits is no longer written (§500). Builds: design 47 G §500, §196, §668 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-watchman/consumes.go | 7 + .../integration_test.go | 4 +- .../cmd/mesh-delivery-watchman/main.go | 26 ++-- .../cmd/mesh-delivery-watchman/main_test.go | 8 + .../internal/watch/busstate.go | 1 + .../internal/watch/findings_test.go | 43 ++---- .../internal/watch/names.go | 3 - .../internal/watch/verdicts.go | 142 +++++++++++------- .../mesh-delivery-watchman/manifest_test.go | 5 +- modules/mesh-delivery-watchman/module.json | 4 +- 10 files changed, 142 insertions(+), 101 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index d176cf4f..e1aa03d0 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -46,6 +46,13 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, return } w.Accepted(a) + case subject == meshcatalog.SubjectManifestRefused: + var r meshcatalog.ManifestRefusedEvent + if err := json.Unmarshal(body, &r); err != nil || r.Module == "" { + logf("a manifest-refused that does not read is left: %v", err) + return + } + w.ManifestRefused(r) case pipelineSubject(w, subject, body): case engineWaitSubject(w, subject, body): case moduleWaitSubject(w, subject, body): diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index faad1c03..b5facb10 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -63,12 +63,12 @@ accounts { } ` -// serveOwners serves the assigner's wants and the indexer's modules as their owners do, by their read tools and views +// serveOwners serves the assigner's wants as its owner does, by their read tools and views // families (DLV-M5, ADR 0615): the watchman (n1) follows both, never their buckets. func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) { t.Helper() viewstest.Serve(t, ctx, js, nc, watch.AssignerModule, wantsState, watch.AssignerModule+".assigner_read_wants", "n1_"+watch.WatchmanModule) - viewstest.Serve(t, ctx, js, nc, watch.IndexerModule, modulesState, watch.IndexerModule+".indexer_read_modules", "n1_"+watch.WatchmanModule) + } // makeEvents makes EVENTS and the watchman's durable consumer of the build seats' events, as the controller and the grants part diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 93b5454c..f2ffe4be 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -194,8 +194,7 @@ func createdOr(e jetstream.KeyValueEntry, at time.Time) time.Time { var firstViewsBucket = watch.WatchmanModule + "_" + views.StatusState const ( - wantsState = "wants" - modulesState = "modules" + wantsState = "wants" // The owners' views families, as their manifests' `streams` name them. ) @@ -261,16 +260,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - modules, err := reader.Follow(watch.IndexerModule, modulesState, watch.IndexerModule+".indexer_read_modules") - if err != nil { - return err - } w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: held, js: js, cred: cred}, time.Now) w.Logf(logger.Printf) // What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end. w.Awaits(watch.AssignerWantsBucket) - w.Awaits(watch.ModulesBucket) // A test shortens how long a move may take; the mesh never sets it. if v := strings.TrimSpace(getenv("MESH_DELIVERY_PULSE_BOUND")); v != "" { d, err := time.ParseDuration(v) @@ -416,6 +410,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("take-in: no connection to ask %s (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err) } else { seedTakeIn(w, calls.Call, logger.Printf) + seedVerdicts(w, calls.Call, logger.Printf) calls.Close() } // The take-in events are the indexer's ordinary events (its emits), heard on the shared events stream through this @@ -452,9 +447,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Every part has registered: the hub pulls the module's durable, and the watchman's other consumers open. hearConsumed(ctx, js, cred, hub, w, logger.Printf) defer hub.Stop() - // The indexer's modules (ADR 0578 §1): a module whose value carries a refusal of its manifest is a finding. - verdictEach := func(e jetstream.KeyValueEntry) { w.ModuleVerdict(e.Key(), e.Value(), opOf(e)) } - go func() { errs <- readLate(ctx, w, modules, watch.ModulesBucket, verdictEach) }() if haveSeen != nil { go func() { errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) @@ -522,6 +514,20 @@ func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMess logf("take-in: %s cannot be asked (%v), so a wait the indexer held before this start is said only once a take-in event of it is heard", pendingTool, err) } +// seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was +// down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after. +func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) { + raw, err := call(watch.VerdictsTool, map[string]any{"all": true}) + if err == nil { + var rows []watch.VerdictRow + if rows, err = watch.DecodeVerdictRows(raw); err == nil { + w.SeedVerdicts(rows) + return + } + } + logf("verdicts: %s cannot be asked (%v), so a manifest refused before this start is said only once a verdict of it is heard", watch.VerdictsTool, err) +} + // takeInHeard hands one take-in event of the indexer to the watchman, by its name relative to the indexer, and acknowledges it; // one that is not a take-in event is said and acknowledged, since hearing it again would not read it. func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 1c662686..7d24b481 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -75,6 +75,14 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Fatalf("created left %v", out.findings) } }) + t.Run("design 47 G §500: the indexer's manifest-refused reaches the watchman by its subject", func(t *testing.T) { + out := &findingsOut{findings: map[string]bool{}} + w := watch.New(out, time.Now) + consumed(w, "mesh.seat.mesh-catalog.event.manifest-refused", []byte(`{"module":"m","digest":"d","problems":["x"]}`), func(string, ...any) {}) + if !out.findings[watch.FindingKey(watch.ScopeModule, "m", watch.KindManifestRefused)] { + t.Fatalf("a refused manifest reached %v", out.findings) + } + }) t.Run("design 47 G §499, §541: a node-engine's wait-held and wait-ended reach the watchman by their subjects", func(t *testing.T) { out := &findingsOut{findings: map[string]bool{}} w := watch.New(out, time.Now) diff --git a/modules/mesh-delivery-watchman/internal/watch/busstate.go b/modules/mesh-delivery-watchman/internal/watch/busstate.go index 11d2d770..2def5880 100644 --- a/modules/mesh-delivery-watchman/internal/watch/busstate.go +++ b/modules/mesh-delivery-watchman/internal/watch/busstate.go @@ -64,6 +64,7 @@ func refusedKey(module, local string) string { func (w *Watchman) Accepted(a meshcatalog.ManifestAcceptedEvent) { w.mu.Lock() defer w.mu.Unlock() + w.manifestAccepted(a.Module, a.Problems) // §500: the problems a stricter validator finds are the module's warning asked := map[string]bool{} for _, e := range a.BusState { if e.Name == "" { diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 6214cee1..0c6f3d11 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -187,43 +187,32 @@ func TestTheWatchmansFindings(t *testing.T) { next := "m.00000000000000000043" w.TakeIn(takeInEvent(t, takein.TakeInPending, next), takeInBody(t, takein.TakeInPending, next, t0)) }, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}}, - {"ADR 0578 §1: a module whose value carries a refusal of its manifest is a finding until it is accepted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - value := func(verdict string, problems ...string) []byte { - b, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: strings.Repeat("b", 64), - Repository: "novox/mesh-catalog", Commit: "c", Verdict: verdict, Problems: problems}) - if err != nil { - t.Fatal(err) - } - return b - } - w.ModuleVerdict("nats", value(meshcatalogclient.VerdictRefused, "removes a stream"), Put) - w.ModuleVerdict("power", value(meshcatalogclient.VerdictRefused, "shows nothing"), Put) + {"design 47 G §500: manifest-refused raises the refusal, manifest-accepted clears it and carries an outgrown manifest's problems", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "nats", Problems: []string{"removes a stream"}}) + w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "power", Problems: []string{"shows nothing"}}) k := FindingKey(ScopeModule, "nats", KindManifestRefused) if f := o.findings[k]; !strings.Contains(f.Summary, "refused the manifest of nats: removes a stream") { t.Fatalf("findings = %v", o.findings) } - w.ModuleVerdict("nats", value(""), Put) - w.ModuleVerdict("power", nil, Delete) + w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "nats", Digest: strings.Repeat("d", 64)}) + w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power", Digest: strings.Repeat("d", 64)}) // ADR 0579 §1: an accepted manifest a stricter judge outgrew is a warning until mended. - outgrown, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: strings.Repeat("b", 64), - Repository: "novox/mesh-tools", Commit: "c", Warnings: []string{"says no credential form"}}) - w.ModuleVerdict("node-tools", outgrown, Put) + w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("d", 64), Problems: []string{"says no credential form"}}) if f := o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)]; f.Severity != Warning || !strings.Contains(f.Summary, "says no credential form") { t.Fatalf("findings = %v", o.findings) } - w.ModuleVerdict("node-tools", value(""), Put) - // Clause 3 of the record on a release's Approve: a manifest that waits for a judge that knows its fields is its own - // warning (a refusal in the value, naming the fields), not manifest-refused, and goes when it is accepted. - waiting, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: strings.Repeat("b", 64), - Repository: "novox/mesh-catalog", Commit: "c", Verdict: meshcatalogclient.VerdictRefused, Problems: []string{"waits"}, Unknown: []string{"core-credentials"}}) - w.ModuleVerdict("nats", waiting, Put) - if f := o.findings[FindingKey(ScopeModule, "nats", KindManifestWaits)]; f.Severity != Warning || !strings.Contains(f.Summary, "core-credentials") { + w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("e", 64), Problems: []string{}}) + // The start seeds from the indexer's verdicts: the newest of each module counts. + w.SeedVerdicts([]VerdictRow{ + {Module: "nats", State: meshcatalogclient.VerdictRefused, Problems: []string{"old"}, At: t0}, + {Module: "nats", State: meshcatalogclient.VerdictAccepted, At: t0.Add(time.Hour)}, + {Module: "power", State: meshcatalogclient.VerdictRefused, Problems: []string{"shows nothing"}, At: t0}, + {Module: "node-tools", State: meshcatalogclient.VerdictAccepted, Warnings: []string{"w"}, At: t0}}) + if len(o.findings) != 2 || o.findings[FindingKey(ScopeModule, "power", KindManifestRefused)].Kind == "" || o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)].Kind == "" { t.Fatalf("findings = %v", o.findings) } - if _, refused := o.findings[FindingKey(ScopeModule, "nats", KindManifestRefused)]; refused { - t.Fatalf("a waiting manifest was found refused: %v", o.findings) - } - w.ModuleVerdict("nats", value(""), Put) + w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power"}) + w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools"}) }, map[string]string{}}, {"DLV-W1, 1966/AC37-AC38 (ADR 0595 §4, design 25 § 12.2): an accepted entry the bus module says nothing of in five minutes is overdue until created; a refused one stands until created; a later verdict no longer asking ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { accepted := func(locals ...string) meshcatalog.ManifestAcceptedEvent { diff --git a/modules/mesh-delivery-watchman/internal/watch/names.go b/modules/mesh-delivery-watchman/internal/watch/names.go index 5759cc5b..d8e9121d 100644 --- a/modules/mesh-delivery-watchman/internal/watch/names.go +++ b/modules/mesh-delivery-watchman/internal/watch/names.go @@ -25,13 +25,10 @@ const ( NodeHealthBucket = "mesh-host_health" // AssignerWantsBucket is the assigner's state wants, followed by its views family (consumes: mesh-delivery-assigner.views.wants). AssignerWantsBucket = AssignerModule + "_wants" - // ModulesBucket is the indexer's state modules, followed by its views family (consumes: mesh-delivery-indexer.views.modules). - ModulesBucket = IndexerModule + "_modules" ) // The findings a value of another part is raised as, named by their words. const ( FindingManifestOutgrown = "manifest-outgrown" - FindingManifestWaits = "manifest-waits" FindingNodeAccountFallback = "node-account-fallback" ) diff --git a/modules/mesh-delivery-watchman/internal/watch/verdicts.go b/modules/mesh-delivery-watchman/internal/watch/verdicts.go index dcb2618c..dfdf8a58 100644 --- a/modules/mesh-delivery-watchman/internal/watch/verdicts.go +++ b/modules/mesh-delivery-watchman/internal/watch/verdicts.go @@ -1,9 +1,13 @@ package watch import ( + "encoding/json" + "fmt" "strings" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) // KindManifestRefused is a module whose manifest the indexer's judge refused (novox/hq ADR 0578 §1): every reader holds @@ -14,66 +18,96 @@ const KindManifestRefused = "manifest-refused" // (novox/hq ADR 0579 §1): it stays accepted and runs; the warning stands until the manifest is mended. const KindManifestOutgrown = FindingManifestOutgrown -// KindManifestWaits is a module whose manifest carries a field the indexer's judge does not know yet (novox/hq ADR 0578 -// §1, amended): every reader holds it as a refused one and keeps what runs, and the warning stands until a judge knows the -// field or the manifest is mended. -const KindManifestWaits = FindingManifestWaits +// The manifest findings are the indexer's verdicts heard as events (design 47 G §500): `manifest-refused` raises the refusal, and +// `manifest-accepted`, which carries `problems` (what a stricter validator finds in a manifest it still accepts), raises +// `manifest-outgrown` or, with none, clears it and any refusal. The watchman seeds both from the indexer's verdicts at its start +// (VerdictsTool). `manifest-waits` is no longer written: every revision is judged by a validator built from its own heads (§16). -// ModuleVerdict is a change of the indexer's value of a module (ModulesBucket): a value carrying a refusal is a -// finding, one that does not (accepted, or not judged yet) or a deleted key ends it. -func (w *Watchman) ModuleVerdict(module string, value []byte, op Op) { +// VerdictsTool is the indexer's tool that lists its verdicts, which seeds the findings at the start. +const VerdictsTool = IndexerModule + ".indexer_verdicts" + +// VerdictRow is one verdict as the tool lists it (the fields the watchman reads). +type VerdictRow struct { + Module string `json:"module"` + State string `json:"state"` + Problems []string `json:"problems"` + Warnings []string `json:"warnings"` + At time.Time `json:"at"` +} + +// ManifestRefused hears the seat mesh-catalog's manifest-refused: the module's manifest is refused until a mended one is accepted. +func (w *Watchman) ManifestRefused(e meshcatalog.ManifestRefusedEvent) { + if e.Module == "" { + w.say("a manifest-refused names no module, left") + return + } w.mu.Lock() defer w.mu.Unlock() - key := FindingKey(ScopeModule, module, KindManifestRefused) - outgrown := FindingKey(ScopeModule, module, KindManifestOutgrown) - waits := FindingKey(ScopeModule, module, KindManifestWaits) - if op != Put { - w.unfind(key) - w.unfind(outgrown) - w.unfind(waits) - return - } - m, err := meshcatalogclient.DecodeModule(value) - if err != nil { - w.say("the indexer's value of %s cannot be read, so whether its manifest is refused is left as it was: %v", module, err) - return - } - v := meshcatalogclient.ModuleVerdict(module, m) - if v != nil && meshcatalogclient.VerdictWaits(*v) { - w.unfind(key) - w.unfind(outgrown) - w.find(Finding{Scope: ScopeModule, ID: module, Kind: KindManifestWaits, Severity: Warning, Source: ModulesBucket, - Summary: meshcatalogclient.VerdictWhy(*v), - Said: strings.Join(v.Unknown, "; "), - Headline: "The manifest of " + module + " waits for a judge that knows its fields", - Explanation: "The manifest carries " + strings.Join(v.Unknown, ", ") + ", which the indexer's judge does not know yet, so every " + - "reader holds " + module + " as a refused one and keeps what runs; an indexer that knows the fields judges it again " + - "at its start, and the newest build of the indexer refuses a field it does not know.", - Needs: "nothing while a newer indexer is on its way; else a pull request that mends the manifest of " + module}) - return - } - w.unfind(waits) - if refused, _ := meshcatalogclient.TakeVerdict(v); !refused { - w.unfind(key) - if v == nil || len(v.Warnings) == 0 { - w.unfind(outgrown) - return + w.verdict(e.Module, e.Problems, nil, true) +} + +// manifestAccepted hears the problems a manifest-accepted carries (Accepted calls it, w.mu held): none clears the module's findings, some are its warning. +func (w *Watchman) manifestAccepted(module string, problems []string) { + w.verdict(module, nil, problems, false) +} + +// SeedVerdicts raises, at the start, the findings of the newest verdict of each module the indexer's tool listed: a refused one, or an +// accepted one with warnings. A module whose newest verdict is accepted clears both. +func (w *Watchman) SeedVerdicts(rows []VerdictRow) { + newest := map[string]VerdictRow{} + for _, r := range rows { + if old, ok := newest[r.Module]; !ok || r.At.After(old.At) { + newest[r.Module] = r } - w.find(Finding{Scope: ScopeModule, ID: module, Kind: KindManifestOutgrown, Severity: Warning, Source: ModulesBucket, - Summary: "the manifest of " + module + " breaks rules the indexer's judge holds it to now: " + strings.Join(v.Warnings, "; "), - Said: strings.Join(v.Warnings, "; "), - Headline: "The manifest of " + module + " needs mending", - Explanation: "A stricter judge found problems in a manifest it accepted before; " + module + " stays accepted and " + - "runs, and its next change is refused until they are mended (novox/hq ADR 0579).", + } + w.mu.Lock() + defer w.mu.Unlock() + for module, r := range newest { + switch r.State { + case meshcatalogclient.VerdictRefused: + w.verdict(module, r.Problems, nil, true) + case meshcatalogclient.VerdictAccepted: + w.verdict(module, nil, r.Warnings, false) + } + } +} + +// DecodeVerdictRows reads the answer of the indexer's verdicts tool. +func DecodeVerdictRows(raw []byte) ([]VerdictRow, error) { + var a struct { + Verdicts []VerdictRow `json:"verdicts"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("not an answer of the indexer's verdicts: %w", err) + } + return a.Verdicts, nil +} + +// verdict writes and clears a module's manifest findings, w.mu held: a refusal (problems, refused) or an outgrown manifest (warnings). +func (w *Watchman) verdict(module string, problems, warnings []string, refused bool) { + refusedKey := FindingKey(ScopeModule, module, KindManifestRefused) + outgrownKey := FindingKey(ScopeModule, module, KindManifestOutgrown) + if refused { + w.unfind(outgrownKey) + w.find(Finding{Scope: ScopeModule, ID: module, Kind: KindManifestRefused, Severity: Warning, Source: "indexer.verdicts", + Summary: "the indexer's judge refused the manifest of " + module + ": " + strings.Join(problems, "; "), + Said: strings.Join(problems, "; "), + Headline: "The manifest of " + module + " is refused", + Explanation: "The indexer's judge refused it, so every node, the grants part and the assigner hold " + module + ": " + + "nothing of it changes until a mended manifest is merged, and every other module goes on.", Needs: "a pull request that mends the manifest of " + module}) return } - w.unfind(outgrown) - w.find(Finding{Scope: ScopeModule, ID: module, Kind: KindManifestRefused, Severity: Warning, Source: ModulesBucket, - Summary: meshcatalogclient.VerdictWhy(*v), - Said: strings.Join(v.Problems, "; "), - Headline: "The manifest of " + module + " is refused", - Explanation: "The indexer's judge refused it, so every node, the grants part and the assigner hold " + module + ": " + - "nothing of it changes until a mended manifest is merged, and every other module goes on.", + w.unfind(refusedKey) + if len(warnings) == 0 { + w.unfind(outgrownKey) + return + } + w.find(Finding{Scope: ScopeModule, ID: module, Kind: KindManifestOutgrown, Severity: Warning, Source: "indexer.verdicts", + Summary: "the manifest of " + module + " breaks rules the indexer's judge holds it to now: " + strings.Join(warnings, "; "), + Said: strings.Join(warnings, "; "), + Headline: "The manifest of " + module + " needs mending", + Explanation: "A stricter judge found problems in a manifest it accepted before; " + module + " stays accepted and " + + "runs, and its next change is refused until they are mended (novox/hq ADR 0579).", Needs: "a pull request that mends the manifest of " + module}) } diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 27f85ec6..5597b29e 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -126,11 +126,11 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 } // The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the // indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's. - if want := []string{"mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_read_modules", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). @@ -176,7 +176,6 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 "health": {watch.NodeHealthBucket, bucket(m.Reads, "mesh-host.health")}, "wants": {watch.AssignerWantsBucket, bucket(m.Consumes, "mesh-delivery-assigner.views.wants")}, "provisions": {watch.ProvisionsBucket, bucket(m.Reads, "mesh-provisioner.provisions")}, - "modules": {watch.ModulesBucket, bucket(m.Consumes, "mesh-delivery-indexer.views.modules")}, "assigner": {watch.AssignerModule + ".assigner_read_wants", entry(m.Invokes, "mesh-delivery-assigner.assigner_read_wants")}, "indexer": {watch.IndexerModule + ".indexer_pending", entry(m.Invokes, "mesh-delivery-indexer.indexer_pending")}, } { diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index d74809c9..eee55b48 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -23,7 +23,7 @@ "invokes": [ "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_pending", - "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions" ], @@ -31,8 +31,8 @@ "mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", - "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", + "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", From 161d6e645118ea007ebce412cd48e9a05c3d1368 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:56:33 +0100 Subject: [PATCH 6/8] WM16: a record is covered only by what is late for the entries of its own revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit record-opened is joined to the take-in.taken of its milestone, ., whose entries name the modules; the record-open finding is left unwritten only while one of those modules is held by an acting-part-late wait (or a store says full), no longer while any wait is late. A record whose revision is not known taken is covered by a standing take-in-waits. Builds: design 47 G §502, A-200 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../internal/watch/pipelines_test.go | 36 +++++++++++++++++++ .../internal/watch/records.go | 25 ++++++++++--- .../internal/watch/signals.go | 3 -- .../internal/watch/takein.go | 10 ++++++ .../internal/watch/waits.go | 2 ++ .../internal/watch/watch.go | 6 +++- 6 files changed, 73 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index f44ff602..230a4739 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -7,6 +7,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) @@ -247,3 +248,38 @@ func TestARecordOpenADayIsAFinding(t *testing.T) { t.Fatal("the pulse of an ended record stands") } } + +// design 47 G §502, A-200 (WM16): a record is tied to the entries of the take-in.taken of its milestone `.`; it +// is covered only while one of those entries is held by an `acting-part-late` wait, so another record's late entry covers nothing. +func TestARecordIsCoveredOnlyByTheWaitsOfItsOwnEntries(t *testing.T) { + w, o, c := setup() + taken := func(key string, source int64, module string) { + var m map[string]any + if err := json.Unmarshal(takeInBody(t, takein.TakeInTaken, key, c.t), &m); err != nil { + t.Fatal(err) + } + m["branch"], m["source"] = "main", source + m["entries"] = []map[string]any{{"commit": "c", "fingerprint": "f", "module": module}} + b, _ := json.Marshal(m) + w.TakeIn(takeInEvent(t, takein.TakeInTaken, key), b) + } + taken("m.1", 7, "web") + taken("m.2", 8, "db") + rec := func(id, milestone string) []byte { + b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, At: &c.t}) + return b + } + w.RecordOpen(rec("rWeb", "main.7")) + w.RecordOpen(rec("rDb", "main.8")) + late, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Node: "ace", Module: "web", Since: c.t, Bound: "30m"}) + w.WaitHeld("mesh-wants", late) + expire(w, c, signalKey(actingRow, "mesh-wants-place-mesh-delivery-indexer-ace-web"), 30*time.Minute) + expire(w, c, signalKey(recordRow, "rWeb"), RecordOpenBound) + expire(w, c, signalKey(recordRow, "rDb"), 0) + if _, ok := o.findings[FindingKey(ScopeDelivery, "rWeb", KindRecordOpen)]; ok { + t.Fatalf("a record whose entry is held by acting-part-late is covered: %v", o.findings) + } + if _, ok := o.findings[FindingKey(ScopeDelivery, "rDb", KindRecordOpen)]; !ok { + t.Fatalf("a record of other entries is open at 24 hours whatever else is late: %v", o.findings) + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/records.go b/modules/mesh-delivery-watchman/internal/watch/records.go index c1ce908d..d08a0e4e 100644 --- a/modules/mesh-delivery-watchman/internal/watch/records.go +++ b/modules/mesh-delivery-watchman/internal/watch/records.go @@ -3,6 +3,7 @@ package watch import ( "encoding/json" "fmt" + "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -93,6 +94,7 @@ func (w *Watchman) RecordOpen(body []byte) { if r.At != nil && !r.At.IsZero() { since = *r.At } + w.recordMilestone[r.Record] = r.Milestone f := Finding{Scope: ScopeDelivery, ID: r.Record, Kind: KindRecordOpen, Severity: Warning, Source: recordRow, Since: since, Summary: fmt.Sprintf("the delivery record of %s has stood open since %s", r.Milestone, stamp(since)), Said: "open for " + RecordOpenBound.String(), Headline: "A delivery is not on every machine yet", @@ -120,17 +122,31 @@ func (w *Watchman) RecordEnded(body []byte) { defer w.mu.Unlock() w.unsignal(recordRow, r.Record) delete(w.opened, r.Record) + delete(w.recordMilestone, r.Record) w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen)) } -// recordCovered says another condition tells the operator why a record is open: a wait past its bound or a full store. -func (w *Watchman) recordCovered() bool { +// recordCovered says another condition tells the operator why a record is open (design 47 G §502, A-200): the record is tied to the +// entries of its source revision's `take-in.taken` by its milestone, `.`, and is covered while one of those +// entries is held by a wait that raised `acting-part-late`, or a store says `full`; a record whose revision is not known to be taken in +// is covered by a standing `take-in-waits`. +func (w *Watchman) recordCovered(record string) bool { + entries, taken := w.takenEntries[w.recordMilestone[record]] for _, v := range w.findings { f, err := DecodeFinding(v) if err != nil { continue } - if f.Kind == KindActingPartLate || (f.Kind == KindStoreFilling && f.Severity == Urgent) { + switch { + case f.Kind == KindStoreFilling && f.Severity == Urgent: + return true + case f.Kind == KindActingPartLate && taken: + for _, module := range entries { + if strings.HasSuffix(f.ID, "-"+module) { + return true + } + } + case f.Kind == KindTakeInWaits && !taken: return true } } @@ -140,11 +156,10 @@ func (w *Watchman) recordCovered() bool { // rejudgeRecords writes the finding of each record past its bound that nothing covers, and ends it when something does. func (w *Watchman) rejudgeRecords() { for id, f := range w.opened { - if w.recordCovered() { + if w.recordCovered(id) { w.unfind(f.Key()) continue } w.find(f) - _ = id } } diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 6db0439f..4485ffc1 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -186,9 +186,6 @@ func (w *Watchman) signalExpired(pk string) { // stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for // unhealthy. func (w *Watchman) stillWrong(haveKey, kind string) bool { - if haveKey == uncovered { - return !w.recordCovered() - } if node, ok := strings.CutPrefix(haveKey, "silent:"); ok { _, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)] return open diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 4482b385..9120b322 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -47,11 +47,21 @@ func (w *Watchman) TakeIn(event string, body []byte) { // The wait names the watched branch it is of (design 47 G §493), by the contract's body. var named meshcatalog.TakeInPending _ = json.Unmarshal(body, &named) + var taken meshcatalog.TakeInTaken + _ = json.Unmarshal(body, &taken) key := t.Key w.mu.Lock() defer w.mu.Unlock() w.takeIn(kind, key, t) w.pendingOf(kind, key, named.Branch) + if kind == takein.TakeInTaken && taken.Branch != "" && taken.Source != 0 && len(taken.Entries) > 0 { + var modules []string + for _, e := range taken.Entries { + modules = append(modules, e.Module) + } + w.takenEntries[fmt.Sprintf("%s.%d", taken.Branch, taken.Source)] = modules // a record's milestone (A-200) + w.rejudgeRecords() + } } // SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every revision diff --git a/modules/mesh-delivery-watchman/internal/watch/waits.go b/modules/mesh-delivery-watchman/internal/watch/waits.go index 235304c9..034bc357 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits.go @@ -46,6 +46,7 @@ func (w *Watchman) WaitEnded(seat string, body []byte) { w.unsignal(actingRow, id) w.unfind(FindingKey(ScopeDelivery, id, KindActingPartLate)) w.unfind(FindingKey(ScopeDelivery, id, KindPutStands)) + w.rejudgeRecords() } func decodeWait(w *Watchman, seat, event string, body []byte) (meshwants.Wait, bool) { @@ -89,6 +90,7 @@ func (w *Watchman) waitHeld(seat string, a meshwants.Wait) { return } w.find(f) + w.rejudgeRecords() // a record whose entry this wait holds is covered now (§502) } // putStands is a put's wait held (`step: put`, design 47 G §498): a put without hold that stands past its bound is said as the diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index bfe049f7..8bcf9ea9 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -120,6 +120,10 @@ type Watchman struct { heads map[string]head // pendingBranch is, by key, the watched branch of each open take-in wait that names one (takein.go, design 47 G §497). pendingBranch map[string]string + // takenEntries are the modules of each taken-in revision, by `.`, and recordMilestone each open record's + // milestone, which joins a record to them (records.go, design 47 G §502). + takenEntries map[string][]string + recordMilestone map[string]string // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). queued map[string]map[string]time.Time // awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go). @@ -132,7 +136,7 @@ type Watchman struct { func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound, - findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, pendingBranch: map[string]string{}, away: map[string]bool{}, + findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, pendingBranch: map[string]string{}, takenEntries: map[string][]string{}, recordMilestone: map[string]string{}, away: map[string]bool{}, haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, From 3eca094c5cb3ccbe4e6f833cbb2b36b89371e399 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:58:55 +0100 Subject: [PATCH 7/8] A-197: the watchman reads no node-engine state: have and health go, running ends a want's pulse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed with the reads of node-engine.have and mesh-host.health: the findings module-unhealthy (module ., token unhealthy, urgent after four hours), kept-as-it-runs (module ., token kept), apply-failing from a have that failed (machine ; the provisioner clients' half stays), machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, directory-used-as-found, node-account-fallback, and the urgency of a silent control node. A want's pulse now ends on the node-engine's running event (consumes node-engine.running). Builds: design 47 G A-197, A-192, §541 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-watchman/consumes.go | 4 +- .../integration_test.go | 13 +- .../cmd/mesh-delivery-watchman/main.go | 49 +- .../internal/watch/findings_test.go | 246 +-------- .../internal/watch/mirror.go | 40 +- .../internal/watch/names.go | 3 - .../internal/watch/nodehealth.go | 485 ------------------ .../internal/watch/signals.go | 202 +------- .../internal/watch/watch.go | 32 +- .../internal/watch/watch_test.go | 18 +- .../mesh-delivery-watchman/manifest_test.go | 6 +- modules/mesh-delivery-watchman/module.json | 5 +- 12 files changed, 68 insertions(+), 1035 deletions(-) delete mode 100644 modules/mesh-delivery-watchman/internal/watch/nodehealth.go diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index e1aa03d0..8dac5cfb 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -200,7 +200,7 @@ func recordSubject(w *watch.Watchman, subject string, body []byte) bool { return true } -// engineWaitSubject hands the node-engine seat's `wait-held` and `wait-ended`, on +// engineWaitSubject hands the node-engine seat's `wait-held`, `wait-ended` and `running`, on // mesh.seat.node-engine.event..., to the watchman (design 47 G §499, §541). It says whether the subject was one. func engineWaitSubject(w *watch.Watchman, subject string, body []byte) bool { parts := strings.Split(subject, ".") @@ -212,6 +212,8 @@ func engineWaitSubject(w *watch.Watchman, subject string, body []byte) bool { w.EngineWaitHeld(parts[5], parts[6], body) case nodeengine.EventWaitEnded: w.EngineWaitEnded(parts[5], parts[6]) + case nodeengine.EventRunning: + w.EngineRunning(parts[5], parts[6], body) default: return false } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index b5facb10..0b08b916 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -17,7 +17,6 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" @@ -150,7 +149,6 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 {Bucket: watch.StuckBucket, History: 20}, {Bucket: watch.PulseBucket, LimitMarkerTTL: time.Second}, {Bucket: watch.AssignerWantsBucket, History: 20}, - {Bucket: watch.HaveBucket, History: 20}, {Bucket: watch.FindingsBucket, History: 5}, {Bucket: firstViewsBucket, History: 1}, } { @@ -390,7 +388,6 @@ func anAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 {Bucket: watch.StuckBucket, History: 20}, {Bucket: watch.PulseBucket, LimitMarkerTTL: time.Second}, {Bucket: watch.AssignerWantsBucket, History: 20}, - {Bucket: watch.HaveBucket, History: 20}, {Bucket: watch.FindingsBucket, History: 5}, {Bucket: firstViewsBucket, History: 1}, } { @@ -472,7 +469,6 @@ func aWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { {Bucket: watch.StuckBucket, History: 20}, {Bucket: watch.PulseBucket, LimitMarkerTTL: time.Second}, {Bucket: watch.AssignerWantsBucket, History: 20}, - {Bucket: watch.HaveBucket, History: 20}, {Bucket: watch.FindingsBucket, History: 5}, {Bucket: firstViewsBucket, History: 1}, } { @@ -498,14 +494,11 @@ func aWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { t.Fatal(err) } } + // The node-engine says `running` on its seat (design 47 G §541), which the watchman hears on its consumer of EVENTS. putHave := func(node, module, fp string) { t.Helper() - body, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - if err != nil { - t.Fatal(err) - } - key, _ := nodeengineclient.HaveKey(node, module) - if _, err := kvs[watch.HaveBucket].Put(ctx, key, body); err != nil { + body, _ := json.Marshal(nodeengine.Running{Fingerprint: fp, Module: module, Node: node, At: time.Now().UTC(), V: 1}) + if _, err := js.Publish(ctx, "mesh.seat.node-engine.event.running."+node+"."+module, body); err != nil { t.Fatal(err) } } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index f2ffe4be..ed02decb 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -221,7 +221,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants // and the node-engine's haves only make the pulses of a move: without the haves the watchman watches presence and the // build seats' steps, and says that moves are not watched (issue 1945). - var kvs struct{ presence, pulse, stuck, haves jetstream.KeyValue } + var kvs struct{ presence, pulse, stuck jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue @@ -230,7 +230,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } } - kvs.haves = process.OptionalBucket(ctx, js, watch.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) // The assigner's wants and the indexer's modules are their owners' states, read by the first view of their read tools and // then their views families, never their buckets (ADR 0615 §1, DLV-M5). An owner that is down never holds the start: each // is read on a goroutine once its first view is answered, and until then the unanswered first view is the watchman's @@ -308,15 +307,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // What the node agents run (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent until the have names its - // fingerprint, so the haves are read at the start and the wants after them, once their first view is answered. - var haveSeen jetstream.KeyWatcher - if kvs.haves != nil { - if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil { - process.NotWatched(logger.Printf, watch.HaveBucket, err, "moves are not watched until the watchman starts again") - haveSeen = nil - } - } // The events first, then the one request of the open connections, so no connection falls between them. onEvent := func(open bool) nats.MsgHandler { return func(m *nats.Msg) { @@ -377,25 +367,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }() }) } - // Each node's own health report (the node-engine's state health): read before the findings are reconciled when the - // bucket is there, bound once it appears when it is not; each change is heard on the watch, never polled. - healthEach := func(e jetstream.KeyValueEntry) { w.NodeHealth(e.Key(), e.Value(), opOf(e)) } - var healthWatch jetstream.KeyWatcher - if kv, err := js.KeyValue(ctx, watch.NodeHealthBucket); err == nil { - if healthWatch, err = settle(ctx, kv, healthEach); err != nil { - process.NotWatched(logger.Printf, watch.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") - } - } else { - process.AwaitBucket(ctx, js, conn, watch.NodeHealthBucket, "no node's own health is said", - logger.Printf).Ready(func(kv jetstream.KeyValue) { - hw, err := settle(ctx, kv, healthEach) - if err != nil { - process.NotWatched(logger.Printf, watch.NodeHealthBucket, err, "no node's own health is said until the watchman starts again") - return - } - go func() { _ = follow(ctx, hw, healthEach) }() - }) - } dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} w.SetRecount(dead.count) dead.count() @@ -439,24 +410,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } errs := make(chan error, 16) - if healthWatch != nil { - go func() { errs <- follow(ctx, healthWatch, healthEach) }() - } // The first views of the wants and the modules, asked now and read each on its own goroutine once answered (ADR 0615). reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "watchman_first_views reads nothing and watchman_first_view_again is not heard", logger.Printf)) // Every part has registered: the hub pulls the module's durable, and the watchman's other consumers open. hearConsumed(ctx, js, cred, hub, w, logger.Printf) defer hub.Stop() - if haveSeen != nil { - go func() { - errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) - }() - go func() { - errs <- readLate(ctx, w, wants, watch.AssignerWantsBucket, func(e jetstream.KeyValueEntry) { - w.Want(e.Key(), e.Value(), opOf(e), createdOr(e, time.Now())) - }) - }() - } + go func() { + errs <- readLate(ctx, w, wants, watch.AssignerWantsBucket, func(e jetstream.KeyValueEntry) { + w.Want(e.Key(), e.Value(), opOf(e), createdOr(e, time.Now())) + }) + }() if provWatch != nil { go func() { errs <- follow(ctx, provWatch, func(e jetstream.KeyValueEntry) { w.Provision(e.Key(), e.Value(), opOf(e)) }) diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 0c6f3d11..6e3a2778 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -3,14 +3,11 @@ package watch import ( "encoding/json" "errors" - "slices" "strings" "sync/atomic" "testing" "time" - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" @@ -30,15 +27,6 @@ func mustEncode(t *testing.T, f Finding) []byte { return v } -func haveOf(t *testing.T, phase string, health string, detail string) []byte { - t.Helper() - b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpNew, Phase: phase, Health: health, Detail: detail}) - if err != nil { - t.Fatal(err) - } - return b -} - // expire hears the expiry of the pulse at pk. func expire(w *Watchman, c *clock, pk string, after time.Duration) { c.t = c.t.Add(after) @@ -323,29 +311,6 @@ func TestTheWatchmansFindings(t *testing.T) { } } }, map[string]string{}}, - {"ADR 0601 §4: what a node keeps of a module as it runs, for a manifest that does not say it yet, is a warning until the have says none", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - have := func(kept ...string) []byte { - b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: "src2:" + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK, Kept: kept}) - if err != nil { - t.Fatal(err) - } - return b - } - k := FindingKey(ScopeModule, "runner-x.ace", "kept") - w.Have("ace.runner-x", have("runner-x runs the tool runner's process here"), Put) - f := o.findings[k] - if f.Kind != KindKeptAsRuns || f.Severity != Warning || !strings.Contains(f.Summary, "the tool runner's process") { - t.Fatalf("findings = %v", o.findings) - } - c.t = c.t.Add(time.Minute) - w.Have("ace.runner-x", have("runner-x runs the tool runner's process here"), Put) - if !o.findings[k].Since.Equal(f.Since) { - t.Errorf("a have written again moved the finding's since: %v, was %v", o.findings[k].Since, f.Since) - } - w.Have("ace.prov-y", have("prov-y carries its own secrets sealed"), Put) - w.Have("ace.runner-x", have(), Put) // its manifest says tool-runner now - w.Have("ace.prov-y", nil, Delete) - }, map[string]string{}}, {"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { askProgress(w, "ask1", t0) expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second) @@ -447,18 +412,6 @@ func TestTheWatchmansFindings(t *testing.T) { } expire(w, c, signalKey("S9", k), AdvisoryQuiet) }, map[string]string{}}, - {"WM3 S21: a heard machine's have failed for one apply interval is apply-failing, naming the package manager; applied again it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Connect("node.ace") - w.Have("ace.web", haveOf(t, nodeengineclient.PhaseFailed, nodeengineclient.HealthUnknown, "refused: the package manager (pacman) does not answer"), Put) - if len(o.findings) != 0 { - t.Fatalf("raised before its bound: %v", o.findings) - } - expire(w, c, signalKey("S21", "ace"), ApplyEvery+time.Second) - if f := o.findings["machine.ace.apply-failing"]; !strings.Contains(f.Summary, "pacman does not answer") || f.Headline != "ace cannot install packages" { - t.Fatalf("findings = %v", o.findings) - } - w.Have("ace.web", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, ""), Put) - }, map[string]string{}}, {"WM3 S21: a provisioner client's failed files are apply-failing past the bound; a silent machine's are not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.Connect("node.ace") w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: ProvFailed, Reason: "disk full", At: t0.Add(-10 * time.Minute)}) @@ -468,48 +421,6 @@ func TestTheWatchmansFindings(t *testing.T) { w.Provisioned(ProvReport{Node: "g14", Module: "web", Status: ProvFailed, Reason: "x", At: t0.Add(-10 * time.Minute)}) w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: "applied", At: t0}) }, map[string]string{}}, - {"WM3: unhealthy once a failed have is confirmed after a minute (written once and steady); recovered within it none; urgent after four hours", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("ace.web", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "the unit failed"), Put) - if len(o.findings) != 0 { - t.Fatalf("raised before its bound: %v", o.findings) - } - w.Have("ace.quick", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "blip"), Put) - w.Have("ace.quick", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, ""), Put) // recovered within the bound - expire(w, c, signalKey("unhealthy-confirm", "quick.ace"), UnhealthyConfirmAfter) - expire(w, c, signalKey("unhealthy-confirm", "web.ace"), UnhealthyConfirmAfter) // the one write, never repeated - if o.findings["module.web.ace.unhealthy"].Severity != Warning || len(o.findings) != 1 { - t.Fatalf("findings = %v", o.findings) - } - expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter) - w.Have("ace.mail", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthWaits, "a part waits for the secret token"), Put) // needs-operator is not observed here - }, map[string]string{"module.web.ace.unhealthy": Urgent}}, - {"WM3: a warning unhealthy finding read back at the start is kept as it was, urgent when its four hours from Since are out", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - seedUnhealthy(t, w, o, Warning, t0.Add(-time.Hour)) - w.Have("ace.web", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "x"), Put) - w.Reconcile() - if o.findings["module.web.ace.unhealthy"].Severity != Warning { - t.Fatalf("cleared: findings %v", o.findings) - } - if ttl := o.ttls[signalKey("unhealthy", "web.ace")]; ttl != UnhealthyUrgentAfter-time.Hour { - t.Fatalf("urgent signal restarted from zero: %v", ttl) - } - if _, ok := w.signals[signalKey("unhealthy-confirm", "web.ace")]; ok { - t.Fatal("a standing finding is confirmed again") - } - expire(w, c, signalKey("unhealthy", "web.ace"), UnhealthyUrgentAfter-time.Hour) - }, map[string]string{"module.web.ace.unhealthy": Urgent}}, - {"WM3: an urgent unhealthy finding read back at the start stays urgent, with no signal restarted", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - seedUnhealthy(t, w, o, Urgent, t0.Add(-5*time.Hour)) - w.Have("ace.web", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "x"), Put) - w.Reconcile() - if o.findings["module.web.ace.unhealthy"].Severity != Urgent || len(w.signals) != 0 { - t.Fatalf("findings %v signals %v", o.findings, w.signals) - } - }, map[string]string{"module.web.ace.unhealthy": Urgent}}, - {"WM3: a restart replays a have already failed: it is confirmed after a minute like a new one", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("novox.fail2ban", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "down since 19:40"), Put) - expire(w, c, signalKey("unhealthy-confirm", "fail2ban.novox"), UnhealthyConfirmAfter) - }, map[string]string{"module.fail2ban.novox.unhealthy": Warning}}, {"1954/D2: a provision in waits is its consumer's needs-operator; provided, it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { enc := func(state, why string) []byte { b, err := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, @@ -528,141 +439,6 @@ func TestTheWatchmansFindings(t *testing.T) { w.Provision("ace.web.oidc", enc(meshprovisionerclient.Waits, "x"), Put) w.Provision("ace.web.oidc", nil, Delete) }, map[string]string{}}, - {"774: a node's own report: its network, the resolver file rewritten, failed units, a relogin, a directory used as found, a login fallen back", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - report := func(h nodeengine.HealthByNodeValue) []byte { - b, err := nodeengineclient.EncodeHealth(h) - if err != nil { - t.Fatal(err) - } - return b - } - w.Have("novox.nats", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, ""), Put) // the control node: the bus node - w.NodeHealth("novox", report(nodeengine.HealthByNodeValue{Node: "novox", Network: &nodeengine.NetworkHealth{State: "unhealthy", Parts: []nodeengine.NetworkPart{ - {Part: "resolv-conf", State: "unhealthy", Writer: "NetworkManager", Owner: "systemd-resolved", Said: "rewritten"}, - {Part: "names", State: "unhealthy", Reason: "names do not resolve.", Said: "x"}, - {Part: "route", State: "unhealthy", Reason: "no default route", Said: "y"}}}, - Units: &nodeengine.UnitsHealth{State: "degraded", Failed: []nodeengine.FailedUnit{{Unit: "cups.service", Scope: "system", Result: "exit-code"}}}, - Relogin: []nodeengine.NodeAccount{{Account: "jochen", Module: "docker", Reason: "a group"}}, - Found: []nodeengine.NodeDirectory{{Module: "photos", Resource: "data", Path: "/srv/photos", Reason: "owner differs"}}, - Fallback: &nodeengine.LoginFallback{Node: "novox", From: "node.novox--b", Why: "refused twice", At: time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC)}}), Put) - for key, sev := range map[string]string{"machine.novox.network": Urgent, "machine.novox.systemd-resolved.rewritten": Urgent, - "machine.novox.units": Warning, "module.docker.novox.relogin-needed": Warning, "module.photos.novox.directory-used-as-found": Warning, - "machine.novox.node-account-fallback": Warning} { - if o.findings[key].Severity != sev { - t.Errorf("%s = %+v", key, o.findings[key]) - } - } - if f := o.findings["machine.novox.network"]; strings.Contains(f.Summary, "names") { - t.Errorf("the names failing through the rewritten file are said twice: %s", f.Summary) - } - w.NodeHealth("novox", report(nodeengine.HealthByNodeValue{Node: "novox"}), Put) // all well again - }, map[string]string{}}, - {"774: a failing part is held at the machines it points at, as the controller holds it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - toward := func(node, part string, targets ...string) []byte { - b, _ := nodeengineclient.EncodeHealth(nodeengine.HealthByNodeValue{Node: node, Network: &nodeengine.NetworkHealth{State: "unhealthy", - Parts: []nodeengine.NetworkPart{{Part: part, State: "unhealthy", Reason: part + " fails", Said: "x", Targets: targets}}}}) - return b - } - // own part: one machine alone failing toward a healthy one is its own. - w.NodeHealth("ace", toward("ace", "names", "dns1"), Put) - if _, ok := o.findings["machine.ace.network"]; !ok || len(o.findings) != 1 { - t.Fatalf("own part: findings %v", o.findings) - } - // two resolvers' machines: a second machine failing toward dns1 holds both there, said once as unreachable. - w.NodeHealth("g14", toward("g14", "names", "dns1"), Put) - if f, ok := o.findings["machine.dns1.unreachable"]; !ok || len(f.Also) != 2 || len(o.findings) != 1 { - t.Fatalf("two toward one resolver: findings %v", o.findings) - } - w.NodeHealth("ace", toward("ace", "names", "dns1", "dns2"), Put) // toward two resolvers, one not down: its own - if _, ok := o.findings["machine.ace.network"]; !ok { - t.Fatalf("two resolvers, one up: findings %v", o.findings) - } - w.NodeHealth("ace", nil, Delete) - w.NodeHealth("g14", nil, Delete) - // the hub silent on the record: a tunnel toward it is held there, and its silence says it. - w.Connect("node.hub") - w.Disconnect("node.hub") - expire(w, c, pulseKeyOf(t, presenceNS, "hub"), 120*time.Second) - w.NodeHealth("lap", toward("lap", "tunnel", "hub"), Put) - if _, own := o.findings["machine.lap.network"]; own { - t.Errorf("a tunnel toward the silent hub is the reporter's own: %v", o.findings) - } - if _, ok := o.findings["machine.hub.unreachable"]; ok { - t.Error("the silent hub is said unreachable as well") - } - // design 47 G §706: the silence lists the machines failing toward it in `also`, as `unreachable` does. - if f := o.findings["machine.hub.silent"]; !slices.Equal(f.Also, []string{"lap"}) { - t.Errorf("the hub's silence lists %v in also", f.Also) - } - w.NodeHealth("lap", nil, Delete) - if f := o.findings["machine.hub.silent"]; len(f.Also) != 0 { - t.Errorf("the hub's silence still lists %v once nothing fails toward it", f.Also) - } - w.NodeHealth("lap", toward("lap", "tunnel", "hub"), Put) - // the hub's own network open, two machines failing toward it: its `network` finding lists both. - w.NodeHealth("hub", toward("hub", "bus"), Put) - w.NodeHealth("ace", toward("ace", "tunnel", "hub"), Put) - if f := o.findings["machine.hub.network"]; !slices.Equal(f.Also, []string{"ace", "lap"}) { - t.Errorf("the hub's network lists %v in also: %v", f.Also, o.findings) - } - w.NodeHealth("hub", nil, Delete) - w.NodeHealth("ace", nil, Delete) - w.Connect("node.hub") // back: lap's tunnel failing toward a hub up is its own again - if _, own := o.findings["machine.lap.network"]; !own { - t.Errorf("hub back: findings %v", o.findings) - } - w.NodeHealth("lap", nil, Delete) - }, map[string]string{}}, - {"774: a node two nodes' tunnels no longer see is unreachable, said once there; not while it is silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - for node, key := range map[string]string{"ace": "kA", "g14": "kG", "lap": "kL"} { - b, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node, TunnelKey: key}}) - w.Have(node+".node", b, Put) - } - peers := func(from string, seen bool) []byte { - b, _ := nodeengineclient.EncodeHealth(nodeengine.HealthByNodeValue{Node: from, Peers: []nodeengine.PeerSeen{{Key: "kL", Seen: seen}}}) - return b - } - w.NodeHealth("ace", peers("ace", false), Put) - if len(o.findings) != 0 { - t.Fatalf("one node alone said it: %v", o.findings) - } - w.NodeHealth("g14", peers("g14", false), Put) - if f := o.findings["machine.lap.unreachable"]; len(f.Also) != 2 { - t.Fatalf("findings = %v", o.findings) - } - w.Connect("node.lap") - w.Disconnect("node.lap") - expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second) - if _, ok := o.findings["machine.lap.unreachable"]; ok { - t.Error("a silent node is also said unreachable") - } - w.Connect("node.lap") - w.NodeHealth("g14", peers("g14", true), Put) - }, map[string]string{}}, - {"WM3: a have removed ends its findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("ace.mail", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "x"), Put) - expire(w, c, signalKey("unhealthy-confirm", "mail.ace"), UnhealthyConfirmAfter) - w.Have("ace.mail", haveOf(t, nodeengineclient.PhaseRemoved, nodeengineclient.HealthUnknown, ""), Put) - }, map[string]string{}}, - {"WM2 S1: a control node silent is urgent after half an hour", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - // 1126 (ADR 0590): the control node is the bus node, the one whose have holds the bus module; a have of the - // ended controller names none. - w.Have("g14.mesh-controller", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, ""), Put) - w.Connect("node.g14") - w.Disconnect("node.g14") - expire(w, c, pulseKeyOf(t, presenceNS, "g14"), 120*time.Second) - if _, ok := w.signals[signalKey("S1", "g14")]; ok { - t.Fatalf("a node running the controller's have is held a control node") - } - w.Have("novox.nats", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthOK, ""), Put) - w.Connect("node.novox") - w.Disconnect("node.novox") - expire(w, c, pulseKeyOf(t, presenceNS, "novox"), 120*time.Second) - if o.findings["machine.novox.silent"].Severity != Warning { - t.Fatalf("findings = %v", o.findings) - } - expire(w, c, signalKey("S1", "novox"), ControlNodeUrgentAfter) - }, map[string]string{"machine.novox.silent": Urgent, "machine.g14.silent": Warning}}, {"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0} w.Closed(KindSlowConsumer, cl) @@ -697,14 +473,6 @@ func TestTheWatchmansFindings(t *testing.T) { t.Fatal("still counted again with nothing held") } }, map[string]string{}}, - {"WM4: resync after a cleared bucket writes the same findings", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.Have("ace.mail", haveOf(t, nodeengineclient.PhaseRunning, nodeengineclient.HealthFailed, "x"), Put) - expire(w, c, signalKey("unhealthy-confirm", "mail.ace"), UnhealthyConfirmAfter) - o.findings = map[string]Finding{} - if n := w.Rewrite(); n != 1 { - t.Fatalf("rewrote %d", n) - } - }, map[string]string{"module.mail.ace.unhealthy": Warning}}, {"WM4: a finding read at the start and not observed again is deleted; a signal's stays", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { for _, f := range []Finding{ {Scope: ScopeDelivery, ID: "x.y", Kind: KindStuck, Severity: Warning, Summary: "old", Source: "stuck"}, @@ -733,7 +501,7 @@ func TestTheWatchmansFindings(t *testing.T) { Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, At: t0}) provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Provided, At: t0}) - w.Have("ace.web", haveValue(t, fpOld), Put) // the node runs what the held want names: the hold is still the wait + w.EngineRunning("ace", "web", []byte(`{"fingerprint":"`+fpOld+`"}`)) // the node runs what the held want names: the hold is still the wait w.Want("want.ace.web", held, Put, t0) w.Provision("ace.photos.oidc", asked, Put) waits := map[string]string{AssignerWantsBucket: "want.ace.web", ProvisionsBucket: "ace.photos.oidc"} @@ -783,14 +551,6 @@ func TestTheWatchmansFindings(t *testing.T) { } } -func seedUnhealthy(t *testing.T, w *Watchman, o *fakeOut, sev string, since time.Time) { - t.Helper() - f := Finding{Scope: ScopeModule, ID: "web.ace", Token: "unhealthy", Kind: KindUnhealthy, Machine: "ace", Severity: sev, Source: "health", Since: since, Summary: "web on ace is not healthy: x"} - v, _ := EncodeFinding(f) - o.findings[f.Key()], _ = DecodeFinding(v) - w.SeedFinding(f.Key(), v) -} - const ( revKey = "m.00000000000000000042" groupKey = "g.feat/take-in" @@ -827,9 +587,9 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte { func TestAFindingIsTheContractsWithItsRefInItsID(t *testing.T) { since := time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) for _, f := range []Finding{ - {Scope: ScopeModule, ID: "web.ace", Token: "unhealthy", Kind: KindUnhealthy, Machine: "ace", Severity: Warning, Source: "health", Since: since, Summary: "s", Said: "said"}, + {Scope: ScopeModule, ID: "web.ace", Token: "unhealthy", Kind: KindStuck, Machine: "ace", Severity: Warning, Source: "stuck", Since: since, Summary: "s", Said: "said"}, {Scope: ScopeModule, ID: "web", Token: KindBusStateOverdue + "-state", Kind: KindBusStateOverdue, Severity: Warning, Source: "bus", Since: since, Summary: "s"}, - {Scope: ScopeMachine, ID: "ace", Kind: KindMachineUnits, Severity: Warning, Source: "units", Since: since, Summary: "s", Needs: "mend"}, + {Scope: ScopeMachine, ID: "ace", Kind: KindSilent, Severity: Warning, Source: "S1", Since: since, Summary: "s", Needs: "mend"}, } { c := f.Contract() wantID := f.ID diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index f20ceefb..a45fc04f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -164,46 +164,30 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { describe("the node agent of %s to run %s at %s", node, module, shortFP(want.Fp)), want.Fp, at, 0) } -// Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the -// fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left. -func (w *Watchman) Have(key string, value []byte, op Op) { - node, module, isNode, err := nodeengineclient.ParseHaveKey(key) +// EngineRunning hears the node-engine of node saying `running` for module (design 47 G §541, A-192, A-197): the fingerprint it runs, or +// that it runs none (stopped). One that names the fingerprint of the node's want for the module ends the want's pulse. +func (w *Watchman) EngineRunning(node, module string, body []byte) { + var r nodeengine.Running + if err := json.Unmarshal(body, &r); err != nil { + w.say("a running of the node-engine of %s for %s does not read, left: %v", node, module, err) + return + } + key, err := nodeengineclient.HaveKey(node, module) if err != nil { return } w.mu.Lock() defer w.mu.Unlock() - if isNode { - // The node's own record: its tunnel key, whose peers' sight of it is network-unreachable. - var record *nodeengine.NodeRecord - if op == Put { - if hn, err := nodeengineclient.DecodeHaveNode(value); err == nil { - record = hn.Record - } - } - w.tunnelKey(node, record) - return - } - switch op { - case Expired: - return - case Delete: + if r.Stopped || r.Fingerprint == "" { delete(w.haves, key) - w.haveFindings(key, node, module, nil) return } - have, err := nodeengineclient.DecodeHave(value) - if err != nil { - w.say("the have %s cannot be read, and is left: %v", key, err) - return - } - w.haves[key] = have.Fp - w.haveFindings(key, node, module, &have) + w.haves[key] = r.Fingerprint wk, err := meshwantsclient.WantKey(node, module) if err != nil { return } - if fp, ok := w.wants[wk]; ok && fp == have.Fp { + if fp, ok := w.wants[wk]; ok && fp == r.Fingerprint { w.moved(AssignerWantsBucket, wk) } } diff --git a/modules/mesh-delivery-watchman/internal/watch/names.go b/modules/mesh-delivery-watchman/internal/watch/names.go index d8e9121d..67060d94 100644 --- a/modules/mesh-delivery-watchman/internal/watch/names.go +++ b/modules/mesh-delivery-watchman/internal/watch/names.go @@ -20,9 +20,6 @@ const ( PulseBucket = WatchmanModule + "_" + meshwatchdogs.StatePulse StuckBucket = WatchmanModule + "_" + meshwatchdogs.StateStuck - // HaveBucket and NodeHealthBucket are the node-engine's states have and health (reads: node-engine.have, mesh-host.health). - HaveBucket = "node-engine_have" - NodeHealthBucket = "mesh-host_health" // AssignerWantsBucket is the assigner's state wants, followed by its views family (consumes: mesh-delivery-assigner.views.wants). AssignerWantsBucket = AssignerModule + "_wants" ) diff --git a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go b/modules/mesh-delivery-watchman/internal/watch/nodehealth.go deleted file mode 100644 index 7295e1b5..00000000 --- a/modules/mesh-delivery-watchman/internal/watch/nodehealth.go +++ /dev/null @@ -1,485 +0,0 @@ -package watch - -import ( - "fmt" - "maps" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" -) - -// A node's own health report (novox/hq ADR 0241, issue 315, the operator's alert table of 2026-10-29): each node's engine -// writes `mesh-host_health.` when what it says changes, and the watchman raises from it what the controller raised -// from the passing health message, under the same keys and words (mesh-control cmd/mesh-controller machine_network.go, -// machine_units.go, module_health.go at e04846e9): -// - machine-network `machine..network`: the machine's own unhealthy parts; urgent on the control node or for the bus; -// - network-rewritten `machine.[.].rewritten`: the resolver file rewritten by another program; -// - network-unreachable `machine..unreachable`: two or more nodes' tunnels no longer see it (their peers' handshake), -// said once there, never while its own network or its silence says it; -// - machine-units `machine..units`: failed units no module places; -// - relogin-needed `module...relogin-needed` and directory-used-as-found -// `module...directory-used-as-found`; -// - node-account-fallback `machine..node-account-fallback`: the engine fell back to its node's old login during a -// rotation (novox/hq ADR 0583 §6), raised while its report carries the fall back and cleared once it no longer does. -// -// A failing part that points at other mesh machines (NetworkPart.Targets: the hub's node, each resolver's node) is held -// there, as the controller holds it: when every machine it points at is down on the record (silent, its own network -// unhealthy, or two machines failing toward it), it is that machine's `unreachable`, said once there, and no finding of -// the reporter's. A tunnel peer two or more nodes no longer see points at its node the same way. - -// The kinds a node's health report raises. -const ( - KindMachineNetwork = "machine-network" - KindNetworkRewritten = "network-rewritten" - KindNetworkUnreachable = "network-unreachable" - KindMachineUnits = "machine-units" - KindReloginNeeded = "relogin-needed" - KindUsedAsFound = "directory-used-as-found" - KindAccountFallback = FindingNodeAccountFallback -) - -// The parts and states a report names (mesh-control internal/link). -const ( - partResolvConf = "resolv-conf" - partNames = "names" - partTunnel = "tunnel" - partBus = "bus" - stateUnhealthy = "unhealthy" -) - -func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } - -// NodeHealth is a change of a node's report: written, or deleted (value nil). -func (w *Watchman) NodeHealth(node string, value []byte, op Op) { - w.mu.Lock() - defer w.mu.Unlock() - if op == Put { - h, err := nodeengineclient.DecodeHealth(value) - if err != nil { - w.say("the health report of %s cannot be read, and is left: %v", node, err) - return - } - w.health[node] = h - } else { - delete(w.health, node) - } - w.nodeFindings(node) - w.networks() -} - -// TunnelKey is a node's own tunnel key, from its record in its have: whose peers see it is read by it. -func (w *Watchman) tunnelKey(node string, record *nodeengine.NodeRecord) { - key := "" - if record != nil { - key = record.TunnelKey - } - if w.tunnels[node] == key { - return - } - if key == "" { - delete(w.tunnels, node) - } else { - w.tunnels[node] = key - } - w.networks() -} - -// nodeFindings writes and deletes one node's own findings from its report. -func (w *Watchman) nodeFindings(node string) { - h, ok := w.health[node] - want := map[string]bool{} - if ok { - if f, raise := unitsOf(node, h.Units); raise { - want[f.Key()] = true - w.find(f) - } - for _, f := range reloginOf(node, h.Relogin) { - want[f.Key()] = true - w.find(f) - } - for _, f := range foundOf(node, h.Found) { - want[f.Key()] = true - w.find(f) - } - if f, raise := fallbackOf(node, h.Fallback); raise { - want[f.Key()] = true - w.find(f) - } - } - for key := range w.findings { - if !want[key] && w.ownsHealthKey(key, node) { - w.unfind(key) - } - } -} - -// ownsHealthKey says a finding key is one of node's own report findings. -func (w *Watchman) ownsHealthKey(key, node string) bool { - f, err := DecodeFinding(w.findings[key]) - if err != nil || f.Machine != node || f.Source != "health-report" { - return false - } - return f.Kind != KindNetworkUnreachable && f.Kind != KindMachineNetwork && f.Kind != KindNetworkRewritten -} - -func (w *Watchman) rewrittenOf(m string, p nodeengine.NetworkPart, all []nodeengine.NetworkPart) Finding { - writer := "" - if p.Writer != "" { - writer = " (" + p.Writer + ")" - } - cost := "the names still resolve through what it wrote" - said := []string{p.Part + ": " + p.Said} - for _, q := range all { - if q.Part == partNames { - cost = strings.TrimSuffix(q.Reason, ".") - said = append(said, q.Part+": "+q.Said) - } - } - id := m - summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ - "back at its next reconcile, or that program gives it back", m, writer, cost) - if p.Owner != "" { - id = m + "." + p.Owner - summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ - "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) - } - severity := Warning - if w.control[m] { - severity = Urgent - } - return Finding{Scope: ScopeMachine, ID: id, Token: "rewritten", Kind: KindNetworkRewritten, Machine: m, Severity: severity, - Source: "health-report", Since: p.Since, Summary: summary, - Said: fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))} -} - -// unitsOf is a machine's failed units no module places, one finding. -func unitsOf(node string, u *nodeengine.UnitsHealth) (Finding, bool) { - if u == nil || len(u.Failed) == 0 { - return Finding{}, false - } - var names, said []string - since := u.Failed[0].Since - for _, f := range u.Failed { - where := "" - if f.Scope == "user" { - where = " (the account's own manager)" - } - names = append(names, f.Unit+where) - scope, result := f.Scope, f.Result - if scope == "" { - scope = "system" - } - if result == "" { - result = "no reason said" - } - line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) - if f.Load != "" && f.Load != "loaded" { - line += ", its unit " + f.Load - } - if f.Resource != "" { - line += ", named by the mesh's own " + f.Resource - } - said = append(said, line+", since "+stampSec(f.Since)) - if f.Since.Before(since) { - since = f.Since - } - } - failed := "1 failed service" - if len(u.Failed) > 1 { - failed = fmt.Sprintf("%d failed services", len(u.Failed)) - } - return Finding{Scope: ScopeMachine, ID: node, Token: "units", Kind: KindMachineUnits, Machine: node, Severity: Warning, - Source: "health-report", Since: since, - Summary: fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ - "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), - Said: strings.Join(said, "; "), - Headline: strings.ToUpper(failed[:1]) + failed[1:] + " on " + node, - Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)}, true -} - -// reloginOf is one finding per module whose accounts wait for a new login. -func reloginOf(node string, accounts []nodeengine.NodeAccount) []Finding { - by := map[string][]nodeengine.NodeAccount{} - for _, a := range accounts { - by[a.Module] = append(by[a.Module], a) - } - var out []Finding - for _, module := range slices.Sorted(maps.Keys(by)) { - var names []string - since := by[module][0].Since - for _, a := range by[module] { - names = append(names, a.Account) - if a.Since.Before(since) { - since = a.Since - } - } - sort.Strings(names) - said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ - "before it was; log out of %[1]s completely and log in again, or restart it", node, orModule(module), strings.Join(names, ", ")) - out = append(out, Finding{Scope: ScopeModule, ID: orModule(module) + "." + node, Kind: KindReloginNeeded, Machine: node, - Severity: Warning, Source: "health-report", Since: since, Summary: said, Said: said}) - } - return out -} - -// foundOf is one finding per module with a directory used as found. -func foundOf(node string, dirs []nodeengine.NodeDirectory) []Finding { - by := map[string][]nodeengine.NodeDirectory{} - for _, d := range dirs { - by[d.Module] = append(by[d.Module], d) - } - var out []Finding - for _, module := range slices.Sorted(maps.Keys(by)) { - var ids, why []string - since := by[module][0].Since - for _, d := range by[module] { - ids = append(ids, d.Resource) - why = append(why, d.Reason) - if d.Since.Before(since) { - since = d.Since - } - } - m := orModule(module) - said := fmt.Sprintf("%s on %s uses %s as found: %s", m, node, strings.Join(ids, ", "), strings.Join(why, "; ")) - out = append(out, Finding{Scope: ScopeModule, ID: m + "." + node, Kind: KindUsedAsFound, Machine: node, Severity: Warning, - Source: "health-report", Since: since, Summary: said, Said: said, - Headline: fmt.Sprintf("%s waits for a directory on %s", m, node), - Explanation: fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ - "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, node, m), - Needs: "hand the directory over from the control-node, as the operator; the details name it and the line to type."}) - } - return out -} - -// fallbackOf is a node's fall back to its old login during a rotation of its account (ADR 0583 §6), one finding. -func fallbackOf(node string, f *nodeengine.LoginFallback) (Finding, bool) { - if f == nil || f.Node != node { - return Finding{}, false - } - said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) - return Finding{Scope: ScopeMachine, ID: node, Token: KindAccountFallback, Kind: KindAccountFallback, Machine: node, Severity: Warning, - Source: "health-report", Since: f.At, Summary: said, Said: said, - Headline: fmt.Sprintf("%s fell back to its old bus login", node), - Needs: fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node)}, true -} - -func orModule(m string) string { - if m == "" { - return "_" - } - return m -} - -// silentAlso sets the machines `also` lists on the open `silent` finding of x, when there is one. -func (w *Watchman) silentAlso(x string, also []string) { - value, ok := w.findings[FindingKey(ScopeMachine, x, KindSilent)] - if !ok { - return - } - f, err := DecodeFinding(value) - if err != nil || slices.Equal(f.Also, also) { - return - } - f.Also = also - w.find(f) -} - -// keepSilentAlso gives a `silent` finding about to be written the `also` the open one holds, so a write of its other -// words (its urgency after half an hour) never drops the machines listed there. -func (w *Watchman) keepSilentAlso(f Finding) Finding { - if f.Kind != KindSilent { - return f - } - if value, ok := w.findings[f.Key()]; ok { - if cur, err := DecodeFinding(value); err == nil { - f.Also = cur.Also - } - } - return f -} - -// pointed is one machine's failing part that points at another machine. -type pointed struct { - from string - part nodeengine.NetworkPart -} - -// networks is every machine's network findings from every report together (the controller's networkObservations): the -// resolver file rewritten, each machine's own unhealthy parts, and the machine others point at, held there. -func (w *Watchman) networks() { - machines := slices.Sorted(maps.Keys(w.health)) - unhealthy := func(m string) []nodeengine.NetworkPart { - h := w.health[m] - if h.Network == nil { - return nil - } - var out []nodeengine.NetworkPart - for _, p := range h.Network.Parts { - if p.State == stateUnhealthy { - out = append(out, p) - } - } - return out - } - targets := func(m string, p nodeengine.NetworkPart) ([]string, bool) { - var out []string - for _, x := range p.Targets { - if x == "" || x == m { - return nil, false - } - if !slices.Contains(out, x) { - out = append(out, x) - } - } - return out, len(out) > 0 - } - pointing := map[string][]pointed{} - for _, m := range machines { - for _, p := range unhealthy(m) { - if xs, ok := targets(m, p); ok { - for _, x := range xs { - pointing[x] = append(pointing[x], pointed{from: m, part: p}) - } - } - } - } - // A tunnel peer a node no longer sees points at that peer's node, as a tunnel part toward it would. - byKey := map[string]string{} - for node, key := range w.tunnels { - byKey[key] = node - } - for _, m := range machines { - for _, p := range w.health[m].Peers { - if x, ok := byKey[p.Key]; ok && x != m && !p.Seen { - pointing[x] = append(pointing[x], pointed{from: m, part: nodeengine.NetworkPart{Part: partTunnel, State: stateUnhealthy, - Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}}) - } - } - } - from := func(x string) []string { - var out []string - for _, pt := range pointing[x] { - if !slices.Contains(out, pt.from) { - out = append(out, pt.from) - } - } - sort.Strings(out) - return out - } - silent := func(x string) bool { - _, ok := w.findings[FindingKey(ScopeMachine, x, KindSilent)] - return ok - } - down := func(x string) bool { return silent(x) || len(unhealthy(x)) > 0 || len(from(x)) >= 2 } - allDown := func(xs []string) bool { - for _, x := range xs { - if !down(x) { - return false - } - } - return len(xs) > 0 - } - // A machine silent on the record holds the failures toward it under its silence, so its finding lists in `also` the - // machines whose failing parts point at it, as `unreachable` does (design 47 G §706); a change of `also` rewrites it. - for key, value := range w.findings { - if f, err := DecodeFinding(value); err == nil && f.Kind == KindSilent && key == f.Key() { - w.silentAlso(f.ID, from(f.ID)) - } - } - want := map[string]bool{} - saysOwn := map[string]bool{} - for _, m := range machines { - parts := unhealthy(m) - if len(parts) == 0 { - continue - } - var own []nodeengine.NetworkPart - var rewritten *nodeengine.NetworkPart - for i, p := range parts { - if p.Part == partResolvConf { - rewritten = &parts[i] - continue - } - if xs, ok := targets(m, p); ok && allDown(xs) { - continue // held at the machines it points at - } - own = append(own, p) - } - if rewritten != nil { - f := w.rewrittenOf(m, *rewritten, parts) - want[f.Key()] = true - w.find(f) - kept := own[:0] - for _, p := range own { - if p.Part != partNames { - kept = append(kept, p) - } - } - own = kept - } - if len(own) == 0 { - continue - } - var words, said []string - severity := Warning - for _, p := range own { - words = append(words, p.Reason) - said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) - if p.Part == partBus { - severity = Urgent - } - } - if w.control[m] { - severity = Urgent - } - summary := fmt.Sprintf("%s's network is not healthy: %s", m, strings.Join(words, "; ")) - if waiting := from(m); len(waiting) > 0 { - severity = Urgent - summary += fmt.Sprintf("; %s cannot reach it", strings.Join(waiting, ", ")) - } - f := Finding{Scope: ScopeMachine, ID: m, Token: "network", Kind: KindMachineNetwork, Machine: m, Also: from(m), Severity: severity, - Source: "health-report", Since: w.health[m].Network.Since, Summary: summary, Said: strings.Join(said, " | ")} - want[f.Key()] = true - saysOwn[m] = true - w.find(f) - } - for _, x := range slices.Sorted(maps.Keys(pointing)) { - if !down(x) || saysOwn[x] || silent(x) { - continue - } - var what, said []string - for _, pt := range pointing[x] { - wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[pt.part.Part] - if wd == "" { - wd = pt.part.Part - } - if !slices.Contains(what, wd) { - what = append(what, wd) - } - said = append(said, fmt.Sprintf("%s: %s: %s", pt.from, pt.part.Part, pt.part.Said)) - } - severity := Warning - if w.control[x] || slices.Contains(what, "the bus on it") { - severity = Urgent - } - f := Finding{Scope: ScopeMachine, ID: x, Token: "unreachable", Kind: KindNetworkUnreachable, Machine: x, Also: from(x), - Severity: severity, Source: "health-report", - Summary: fmt.Sprintf("%s cannot be reached from %s: %s", x, strings.Join(from(x), ", "), strings.Join(what, ", ")), - Said: strings.Join(said, " | ")} - want[f.Key()] = true - w.find(f) - } - for key, value := range w.findings { - f, err := DecodeFinding(value) - if err != nil || f.Source != "health-report" { - continue - } - if (f.Kind == KindMachineNetwork || f.Kind == KindNetworkRewritten || f.Kind == KindNetworkUnreachable) && !want[key] { - w.unfind(key) - } - } -} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 4485ffc1..24361bf1 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -8,8 +8,6 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" ) @@ -45,14 +43,8 @@ const ( HeartbeatEvery = 60 * time.Second // ApplyEvery is one apply interval (S21): a failure still there after it is not one the next apply clears. ApplyEvery = 5 * time.Minute - // UnhealthyConfirmAfter is how long a have must keep saying failed before the module is said unhealthy. - UnhealthyConfirmAfter = time.Minute - // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. - UnhealthyUrgentAfter = 4 * time.Hour - // SilentBound is the bound S1 says, three of the intervals a machine has always used, and ControlNodeUrgentAfter - // how long a control node may be silent before it is urgent (the controller's silentBound, controlNodeUrgentAfter). - SilentBound = 3 * HeartbeatEvery - ControlNodeUrgentAfter = 30 * time.Minute + // SilentBound is the bound S1 says, three of the intervals a machine has always used (the controller's silentBound). + SilentBound = 3 * HeartbeatEvery ) // signalBucket is the pseudo-bucket of a signal's pulse key. @@ -179,27 +171,16 @@ func (w *Watchman) signalExpired(pk string) { if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) { return } - w.find(w.keepSilentAlso(due.Finding)) + w.find(due.Finding) } } -// stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for -// unhealthy. -func (w *Watchman) stillWrong(haveKey, kind string) bool { - if node, ok := strings.CutPrefix(haveKey, "silent:"); ok { +// stillWrong says a silent machine's finding still stands, which a row's finding waited on. +func (w *Watchman) stillWrong(key, _ string) bool { + if node, ok := strings.CutPrefix(key, "silent:"); ok { _, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)] return open } - h, ok := w.haveState[haveKey] - if !ok { - return false - } - switch kind { - case KindApplyFailing: - return h.Phase == nodeengineclient.PhaseFailed - case KindUnhealthy: - return h.Health == nodeengineclient.HealthFailed - } return false } @@ -212,10 +193,8 @@ const ( KindSelfCheckSilent = "self-check-silent" KindToolsSilent = "tools-silent" KindApplyFailing = "apply-failing" - KindUnhealthy = "module-unhealthy" KindNeedsOperator = "needs-operator" KindDeadLetters = "max-deliveries" - KindKeptAsRuns = "kept-as-it-runs" ) func stamp(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") } @@ -264,7 +243,6 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) { } else { w.machineAway(node, s.Since) } - defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence if s.Moved || w.away[node] { w.unfind(key) w.unsignal("S1", node) @@ -274,18 +252,7 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) { f := Finding{Scope: ScopeMachine, ID: node, Kind: KindSilent, Machine: node, Severity: Warning, Source: "S1", Summary: fmt.Sprintf("%s has not been heard from since %s (bound %s)", node, stamp(s.Since), SilentBound), Said: fmt.Sprintf("offline since %s", s.Since.UTC().Format(time.RFC3339)), Since: s.Since} - // A control node, the bus node (ControlNodeModule), is urgent after half an hour silent, as the controller - // says it (its watchHeartbeats): the expiry of a pulse from its silence's start, never a loop. - if w.control[node] { - if age := w.now().Sub(s.Since); age > ControlNodeUrgentAfter { - f.Severity = Urgent - } else { - u := f - u.Severity = Urgent - w.signal("S1", node, ControlNodeUrgentAfter-age+time.Second, signalDue{Do: dueFind, While: "silent:" + node, Finding: u}) - } - } - w.find(w.keepSilentAlso(f)) + w.find(f) w.applyCheck(node) return } @@ -478,114 +445,7 @@ func ToolsNode(subject string) string { return node } -// ---- S21 and unhealthy, from the haves and the provisioner clients' reports ---- - -// haveFindings reads a have's change for the rows the haves feed: S21 (the machine's applies failing, applyCheck), -// unhealthy (two writes in a row, urgent after four hours), and where the bus module runs (S1's control node). A have -// removed or deleted ends its part of them. -// -// needs-operator is not observed from the haves: a module's part that waits for the operator is a provision in the state -// `waits` of the seat mesh-provisioner (Provision, ADR 0570 §7). -func (w *Watchman) haveFindings(key string, node, module string, h *nodeengine.HaveByNodeModuleValue) { - unhealthyKey := FindingKey(ScopeModule, module+"."+node, "unhealthy") - if module == ControlNodeModule { - w.control[node] = h != nil && h.Phase != nodeengineclient.PhaseRemoved - } - if h == nil || h.Phase == nodeengineclient.PhaseRemoved { - delete(w.haveState, key) - delete(w.unhealthy, key) - delete(w.failedSince, key) - w.unsignal("unhealthy-confirm", module+"."+node) - w.unsignal("unhealthy", module+"."+node) - w.unfind(unhealthyKey) - w.keptFinding(key, node, module, nil) - w.applyCheck(node) - return - } - w.haveState[key] = *h - now := w.now() - w.keptFinding(key, node, module, h.Kept) - - // S21: a have refused or applied in part says phase failed, with the node's words in its detail. - if h.Phase == nodeengineclient.PhaseFailed { - if _, ok := w.failedSince[key]; !ok { - w.failedSince[key] = now - } - } else { - delete(w.failedSince, key) - } - w.applyCheck(node) - - // unhealthy: a have that says failed and still says it after UnhealthyConfirmAfter (ADR 0564 §4's confirmation, by time: - // a have written once and then unchanged never gets a second write). The confirm signal's expiry writes the warning; a - // signal already standing (kept across a restart) is not renewed, so a steady have does not push its own expiry out. - if h.Health == nodeengineclient.HealthFailed { - if w.unhealthy[key] == 0 { - w.unhealthy[key] = 1 - f := Finding{Scope: ScopeModule, ID: module + "." + node, Token: "unhealthy", Kind: KindUnhealthy, Machine: node, - Severity: Warning, Source: "health", Since: now, - Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, orWords(h.Detail, "its check failed")), - Said: firstLine(orWords(h.Detail, "failed")), - Headline: fmt.Sprintf("%s not working on %s", module, node)} - id := module + "." + node - urgentIn := UnhealthyUrgentAfter - if w.seededFindings[unhealthyKey] { - // Read back at the start: it stands, so it is observed again, kept as it was (not cleared and raised again), and - // its four hours run on from its own Since, not from zero; one that already went urgent needs no signal. - delete(w.seededFindings, unhealthyKey) - old, err := DecodeFinding(w.findings[unhealthyKey]) - if err == nil && old.Severity == Urgent { - urgentIn = 0 - } else if err == nil && !old.Since.IsZero() { - urgentIn = max(time.Second, UnhealthyUrgentAfter-now.Sub(old.Since)) - } - } else if _, ok := w.signals[signalKey("unhealthy-confirm", id)]; !ok { - w.signal("unhealthy-confirm", id, UnhealthyConfirmAfter, signalDue{Do: dueFind, While: key, Finding: f}) - } - if _, ok := w.signals[signalKey("unhealthy", id)]; !ok && urgentIn > 0 { - f.Severity = Urgent - w.signal("unhealthy", id, urgentIn, signalDue{Do: dueFind, While: key, Finding: f}) - } - } - } else { - delete(w.unhealthy, key) - w.unsignal("unhealthy-confirm", module+"."+node) - w.unsignal("unhealthy", module+"."+node) - w.unfind(unhealthyKey) - } -} - -// keptFinding raises, as a warning, what a node keeps of a module as it runs because its manifest does not yet say what shared -// code no longer infers (its have's Kept, novox/hq ADR 0601 §4: the tool runner, the core's credentials), naming each; and -// clears it once the have says none, or is gone. Its since is the first have that said it, so a have written again moves -// nothing. -func (w *Watchman) keptFinding(key, node, module string, kept []string) { - id := module + "." + node - fkey := FindingKey(ScopeModule, id, "kept") - if len(kept) == 0 { - delete(w.keptSince, key) - w.unfind(fkey) - return - } - if w.keptSince == nil { - w.keptSince = map[string]time.Time{} - } - since, ok := w.keptSince[key] - if !ok { - since = w.now() - w.keptSince[key] = since - } - words := strings.Join(kept, "; ") - w.find(Finding{Scope: ScopeModule, ID: id, Token: "kept", Kind: KindKeptAsRuns, Machine: node, Severity: Warning, - Source: "node-engine", Since: since, Summary: fmt.Sprintf("%s on %s is kept as it runs: %s", module, node, words), - Said: firstLine(words), Headline: fmt.Sprintf("%s kept as it runs on %s", module, node), - Needs: "a build of " + module + " whose manifest says it (tool-runner, core-credentials)"}) -} - -// ControlNodeModule is the module whose have names a control node: the bus module, so the control node is the bus node, -// the one stably reachable node every part depends on (novox/hq ADR 0590, issue 1126: the controller that named it ended). -// The same name the bus module's events are heard under (SlowConsumerSubject). -const ControlNodeModule = "nats" +// ---- S21, from the provisioner clients' reports ---- // Provision is a change of a key of mesh-provisioner's `provisions` (ADR 0570 §7): a provision in `waits` is a part of its // consumer waiting for the operator, said as `module...needs-operator` naming each waiting provision and @@ -713,24 +573,12 @@ func (w *Watchman) heard(node string) bool { func (w *Watchman) applyCheck(node string) { key := FindingKey(ScopeMachine, node, KindApplyFailing) type failure struct{ module, words string } - var applied, placed []failure + var placed []failure var since time.Time - for hk, at := range w.failedSince { - n, module, _, err := nodeengineclient.ParseHaveKey(hk) - if err != nil || n != node { - continue - } - applied = append(applied, failure{module, firstLine(orWords(w.haveState[hk].Detail, "no reason given"))}) - if since.IsZero() || at.Before(since) { - since = at - } - } - if len(applied) == 0 { - for module, r := range w.provFailed[node] { - placed = append(placed, failure{module, firstLine(orWords(r.Reason, "no reason given"))}) - if since.IsZero() || r.At.Before(since) { - since = r.At - } + for module, r := range w.provFailed[node] { + placed = append(placed, failure{module, firstLine(orWords(r.Reason, "no reason given"))}) + if since.IsZero() || r.At.Before(since) { + since = r.At } } if since.IsZero() || !w.heard(node) { @@ -745,7 +593,7 @@ func (w *Watchman) applyCheck(node string) { } return } - failed := append(applied, placed...) + failed := placed sort.Slice(failed, func(i, j int) bool { return failed[i].module < failed[j].module }) var names, words []string tool := "" @@ -761,19 +609,14 @@ func (w *Watchman) applyCheck(node string) { } f := Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: since, Said: strings.Join(words, "; "), Headline: node + " is not applying its update"} - switch { - case len(applied) == 0: - f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node, - strings.Join(names, ", "), stamp(since), ApplyEvery) - f.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s. The details say why.", - node, strings.Join(names, ", ")) - case tool != "": - f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, "+ + f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node, + strings.Join(names, ", "), stamp(since), ApplyEvery) + f.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s. The details say why.", + node, strings.Join(names, ", ")) + if tool != "" { + f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is placed, "+ "since %s (bound %s)", node, tool, stamp(since), ApplyEvery) f.Headline = node + " cannot install packages" - default: - f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", node, strings.Join(names, ", "), stamp(since), ApplyEvery) - f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", node) } w.find(f) } @@ -931,13 +774,12 @@ func orWords(s, otherwise string) string { // SignalRows are the rows the watchman holds, in the table's order, for `signals`. var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ - {"S1", "a machine's presence (not while it said it sleeps or shuts down); a control node urgent after " + ControlNodeUrgentAfter.String(), SilentBound.String(), KindSilent}, + {"S1", "a machine's presence (not while it said it sleeps or shuts down)", SilentBound.String(), KindSilent}, {"S8", "a provider's failing word repeated", ProviderSaysAgainWithin.String(), KindProviderSilent}, {"S9", "the bus's advisories", "any occurrence; clears after " + AdvisoryQuiet.String() + " without another", "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost"}, {"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent}, {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, - {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, - {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, + {"S21", "a machine's provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, {"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits}, {actingRow, "an acting part's wait held, or a put's, a put without hold that stands (the seats' wait-held, ended by wait-ended)", "the wait's own bound", KindActingPartLate + ", " + KindPutStands}, {storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling}, diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 8bcf9ea9..4213a57f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -8,7 +8,6 @@ import ( "sync" "time" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -64,10 +63,8 @@ type Watchman struct { presence map[string]presence pulses map[string]*pulse // live pulses, by pulse key stuck map[string]meshwatchdogsclient.Stuck - // keptSince is, by have key, when its have first said what the node keeps of the module as it runs (keptFinding). - keptSince map[string]time.Time - wants map[string]string // the fingerprint of each want that stands, by want key - haves map[string]string // the fingerprint each node runs of a module, by have key + wants map[string]string // the fingerprint of each want that stands, by want key + haves map[string]string // the fingerprint each node runs of a module, by have key, from the node-engine's running // starts is, by pulse key, the first start of a wait whose mark still stands (mirrorKept). starts map[string]startMark held map[string]bool // each want key held by a rule (ADR 0603 §8), mirrored as the hold's wait @@ -85,24 +82,15 @@ type Watchman struct { takeInCaught bool // away is each machine that said it sleeps or shuts down, until it says it is back. away map[string]bool - // haveState is each module's have as last read, unhealthy its writes in a row that said it failed, advised each - // advisory's count while its finding stands and advisedSince since when. - haveState map[string]nodeengine.HaveByNodeModuleValue - unhealthy map[string]int + // advised is each advisory's count while its finding stands and advisedSince since when. advised map[string]int advisedSince map[string]time.Time - // control is each node running the controller; failedSince each have's first failure still standing; provFailed each - // machine's provisioner client's failed reports; closures each bus closure counted, by finding key and identity. - control map[string]bool - failedSince map[string]time.Time - provFailed map[string]map[string]ProvReport - closures map[string]map[string]time.Time - recount func() + // provFailed is each machine's provisioner client's failed reports; closures each bus closure counted, by finding key and identity. + provFailed map[string]map[string]ProvReport + closures map[string]map[string]time.Time + recount func() // waiting is, by `.`, each of its provisions in waits, with its words. waiting map[string]map[string]string - // health is each node's own health report, and tunnels each node's tunnel key from its record. - health map[string]nodeengine.HealthByNodeValue - tunnels map[string]string // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration // progressBound is how long a running ask may say nothing of its progress; a test shortens it. @@ -137,9 +125,9 @@ func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, pendingBranch: map[string]string{}, takenEntries: map[string][]string{}, recordMilestone: map[string]string{}, away: map[string]bool{}, - haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, - control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, - health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, + advised: map[string]int{}, advisedSince: map[string]time.Time{}, + provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 3862f84b..0a8e05ea 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -8,8 +8,6 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" @@ -614,14 +612,8 @@ func wantValue(t *testing.T, fp string, removed bool) []byte { return b } -func haveValue(t *testing.T, fp string) []byte { - t.Helper() - b, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - if err != nil { - t.Fatal(err) - } - return b -} +// runningBody is the node-engine's `running` for a build of fingerprint fp (design 47 G §541). +func runningBody(fp string) []byte { return []byte(`{"fingerprint":"` + fp + `"}`) } var ( fpOld = meshcatalogclient.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -637,11 +629,11 @@ func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) { if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") { t.Fatalf("pulses = %v", o.pulses) } - w.Have("ace.web", haveValue(t, fpOld), Put) // the node still runs the old one + w.EngineRunning("ace", "web", runningBody(fpOld)) // the node still runs the old one if _, ok := o.pulses[pk]; !ok { t.Fatalf("a have of another fingerprint ended the pulse") } - w.Have("ace.web", haveValue(t, fpNew), Put) + w.EngineRunning("ace", "web", runningBody(fpNew)) if _, ok := o.pulses[pk]; ok { t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses) } @@ -653,7 +645,7 @@ func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) { func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) { w, o, c := setup() c.t = t0.Add(time.Second) - w.Have("ace.web", haveValue(t, fpNew), Put) + w.EngineRunning("ace", "web", runningBody(fpNew)) w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0) w.Want("want.ace.mail", wantValue(t, "", true), Put, t0) if len(o.pulses) != 0 { diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 5597b29e..d78bcf2e 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -103,7 +103,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 reads := slices.Clone(m.Reads) slices.Sort(reads) // The build seats' steps are no read (design 47 G §483, A-136): the watchman times an ask from the seats' step-progress. - if !reflect.DeepEqual(reads, []string{"mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { + if !reflect.DeepEqual(reads, []string{"mesh-provisioner.provisions"}) { t.Errorf("reads = %v", reads) } // Design 47 G §368: the version is MAJOR.MINOR and nothing else. @@ -130,7 +130,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). @@ -172,8 +172,6 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 "presence": {watch.PresenceBucket, m.Module + "_" + m.State[0].Name}, "pulse": {watch.PulseBucket, m.Module + "_" + m.State[1].Name}, "stuck": {watch.StuckBucket, m.Module + "_" + m.State[2].Name}, - "have": {watch.HaveBucket, bucket(m.Reads, "node-engine.have")}, - "health": {watch.NodeHealthBucket, bucket(m.Reads, "mesh-host.health")}, "wants": {watch.AssignerWantsBucket, bucket(m.Consumes, "mesh-delivery-assigner.views.wants")}, "provisions": {watch.ProvisionsBucket, bucket(m.Reads, "mesh-provisioner.provisions")}, "assigner": {watch.AssignerModule + ".assigner_read_wants", entry(m.Invokes, "mesh-delivery-assigner.assigner_read_wants")}, diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index eee55b48..e113c701 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -16,9 +16,7 @@ } ], "reads": [ - "mesh-host.health", - "mesh-provisioner.provisions", - "node-engine.have" + "mesh-provisioner.provisions" ], "invokes": [ "mesh-delivery-assigner.assigner_read_wants", @@ -70,6 +68,7 @@ "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", + "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-conditions.findings-asked" From edc8bbd9ab27abfefb6eb4005eccc92880e7dde3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:59:49 +0100 Subject: [PATCH 8/8] =?UTF-8?q?=C2=A77,=20=C2=A7428=20(watchman):=20a=20bu?= =?UTF-8?q?s=20user=20is=20said=20by=20its=20module's=20name,=20the=20tool?= =?UTF-8?q?=20runner's=20no=20differently?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit signals.go BusUserWords: the node-tools special word is gone (the node-engine's node. user stays, a bus account form, not a module name). The watchman's ControlNodeModule (signals.go:591, nats) and builderModule (names.go:17) went with A-197 and WM13. Builds: design 47 G §7, §428 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-watchman/internal/watch/signals.go | 3 --- .../internal/watch/watch_test.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 24361bf1..115687ed 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -712,9 +712,6 @@ func BusUserWords(user string) (string, string) { return "the node-engine on " + m, m } if m, module, ok := strings.Cut(user, "."); ok { - if module == "node-tools" { - return "the tool runner on " + m, m - } return module + " on " + m, m } return "a client of the bus", "" diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 0a8e05ea..591d1748 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -745,3 +745,13 @@ func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) { }) } } + +// design 47 G §7, §428: no part's code names another module: a bus user is said by its own module's name on its machine, the tool +// runner's no differently from any module's. +func TestABusUserIsSaidByItsModuleWithoutAModulesNameInTheCode(t *testing.T) { + for user, want := range map[string]string{"ace.node-tools": "node-tools on ace", "ace.web": "web on ace", "node.ace": "the node-engine on ace"} { + if got, _ := BusUserWords(user); got != want { + t.Errorf("BusUserWords(%q) = %q, want %q", user, got, want) + } + } +}