package main import ( "context" "errors" "os" "path/filepath" "regexp" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is // checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound — // nothing raised — and just past it — its condition raised, with its kind and severity — and restored, // and the condition clears. A row that is not watched says why. A row added to the table without a // suppression here fails, so the table cannot grow a watchdog nobody has seen fire. // calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send // reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call // running, the self-check a minute old. func calm(now time.Time) *signalFacts { return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour), machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second), every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute), lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}}, plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2, entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}}, loop: loopFacts{took: now.Add(-time.Second), pending: 1}, mergesPassed: now.Add(-time.Minute), selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute}, lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{}, lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)}, facts: factsFacts{taken: now.Add(-time.Hour), began: now.Add(-time.Hour)}, } } // refusedBy is n refusals of one writer, the last a moment ago. func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals { return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n, Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}} } // suppression is one row's signal held back: inside its bound, and past it. type suppression struct{ inside, past func(f *signalFacts) } // suppressions are every watched row's, by row. var suppressions = map[string]suppression{ "S1": { inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) }, past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) }, }, "S2": { inside: func(f *signalFacts) { f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false }, past: func(f *signalFacts) { f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false }, }, "S3": { inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) }, past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) }, }, "S4": { inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) }, past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) }, }, "S5": { inside: func(f *signalFacts) {}, past: func(f *signalFacts) { f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}} }, }, "S6": { inside: func(f *signalFacts) { f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight, on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}} }, past: func(f *signalFacts) { f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight, on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}} }, }, "S7": { inside: func(f *signalFacts) { f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}} }, past: func(f *signalFacts) { f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}} }, }, "S8": { inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} }, past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} }, }, // A message given up on and not kept: the one max-deliveries advisory S9 says itself (issue 440). "S9": { inside: func(f *signalFacts) { f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}} }, past: func(f *signalFacts) { f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}} }, }, "S10": { inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) }, past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) }, }, "S11": { inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) }, past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) }, }, "S12": { inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) }, past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) }, }, "S13": { inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) }, past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) }, }, // The snapshot a merge check is fed, older than two days; or none kept by a controller two days up. "S14": { inside: func(f *signalFacts) { f.facts.taken = f.now.Add(-47 * time.Hour) }, past: func(f *signalFacts) { f.facts.taken = f.now.Add(-49 * time.Hour) }, }, // A walk waiting for its delivery's word for 30 minutes (novox/hq ADR 0239). "S16": { inside: func(f *signalFacts) { f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", since: f.now.Add(-29 * time.Minute)}} }, past: func(f *signalFacts) { f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", since: f.now.Add(-31 * time.Minute)}} }, }, // A batch still assembling past its maximum with no walk open (novox/hq ADR 0276): a minute's grace. "S18": { inside: func(f *signalFacts) { f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", atMost: f.now.Add(-59 * time.Second), closed: f.now.Add(-59 * time.Second)}} }, past: func(f *signalFacts) { f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", atMost: f.now.Add(-61 * time.Second), closed: f.now.Add(-61 * time.Second)}} }, }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11", closed: f.now.Add(-59 * time.Minute), behind: "plan-1", walkBound: time.Hour}} }, past: func(f *signalFacts) { f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11", closed: f.now.Add(-61 * time.Minute), behind: "plan-1", walkBound: time.Hour}} }, }, // A send refused because it would replace the bus outside its planned step: said at its first refusal, // whatever the bound (novox/hq issue 336). Inside: a new bus build waits, and no send was refused for it. "S17": { inside: func(f *signalFacts) { f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"}, machines: []string{"anchor"}} }, past: func(f *signalFacts) { f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"}, machines: []string{"anchor"}, waits: []busWaitFacts{{plan: "plan-1", repository: "novox/app", commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}} }, }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"), actByHand(f.now.Add(-time.Hour), "consumer-behind")} }, past: func(f *signalFacts) { f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"), actByHand(f.now.Add(-time.Hour), "consumer-behind")} }, }, } // actByHand is one entry in the hand-act log, with its cause. func actByHand(at time.Time, cause string) link.HandAct { return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop", Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause} } // **S15 names the cause and, where a healer answers it, that the healer was not enough.** func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) f := calm(now) f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"), actByHand(now.Add(-time.Minute), "restarted the proxy")} got := watchHandActs(f) if len(got) != 2 { t.Fatalf("%+v", got) } if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") { t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary) } if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" || !strings.Contains(got[1].Summary, "a healer is wanted") { t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary) } } // actOf is an act in the log recorded by a verb, with its cause. func actOf(at time.Time, verb, cause string) link.HandAct { a := actByHand(at, cause) a.Verb, a.Args = verb, nil return a } // **An act a person decides by design is no repair** (handActVerbs): approving or rejecting a // retirement and deleting what was retired (ADR 0230), a bus upgrade and a backlog released (ADR 0236), // and a rotation after a leak — repeated, none is a healer wanted, whatever cause it gives. func TestAPersonsDecisionRepeatedWantsNoHealer(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) for _, c := range []struct{ verb, cause string }{ {"retire approve", kindRetireWaiting}, {"retire reject", kindRetireWaiting}, {"cleanup delete", kindCleanupWaiting}, {"bus upgrade", "bus-upgrade"}, {"bus upgrade", "a word the person chose"}, {"upgrade release-backlog", "upgrade release-backlog"}, {"secret rotate", causeLeakedInLogs}, } { f := calm(now) f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause), actOf(now.Add(-time.Hour), c.verb, c.cause)} if got := watchHandActs(f); len(got) != 0 { t.Errorf("%s (cause %s) repeated asked for a healer: %+v", c.verb, c.cause, got) } } } // **What repairs still counts**: a push by hand above all (ADR 0236 exists to end it), a rotation for // any cause but a leak, an act recorded outside the mesh whatever cause it names, and a verb the table // does not know. func TestARepairRepeatedStillWantsAHealer(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) for _, c := range []struct{ verb, cause string }{ {"push", "push"}, {"plans close", "plans close"}, {"conditions silence", "consumer-behind"}, {"secret rotate", "stopped-working"}, {"hand-act record", "bus-upgrade"}, {"hand-act record", kindCleanupWaiting}, {"a verb nobody listed", "x"}, } { f := calm(now) f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause), actOf(now.Add(-time.Hour), c.verb, c.cause)} if got := watchHandActs(f); len(got) != 1 || got[0].Kind != "healer-wanted" { t.Errorf("%s (cause %s) repeated wanted no healer: %+v", c.verb, c.cause, got) } } // A decision does not make up the second of a cause a repair recorded once. f := calm(now) f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act record", "bus-upgrade"), actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")} if got := watchHandActs(f); len(got) != 0 { t.Errorf("one repair and one decision asked for a healer: %+v", got) } } // **A healer-wanted already open for a decision clears on the next tick** — the log of 2026-10-06: // a bus upgrade recorded after the fact and one through its verb, and two rotations after a leak. func TestAHealerWantedOpenForADecisionClearsOnTheNextTick(t *testing.T) { now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC) store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}, Now: func() time.Time { return now }}) defer k.Close(context.Background()) // What the build before this one raised, as it raised it. var before []conditions.Observation for _, cause := range []string{"bus-upgrade", causeLeakedInLogs} { before = append(before, conditions.Observation{Scope: conditions.ScopeMesh, ID: "hand-acts." + cause, Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning, Summary: "repaired twice"}) } if err := k.Reconcile(t.Context(), "S15", before); err != nil { t.Fatal(err) } if open, _ := k.Open(t.Context()); len(open) != 2 { t.Fatalf("the conditions of the build before were not open: %+v", open) } f := calm(now) f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", "bus-upgrade"), actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs), actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs), actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")} w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} w.see(t.Context(), f) if open, _ := k.Open(t.Context()); len(open) != 0 { t.Fatalf("a healer-wanted for a person's decision stayed open: %+v", open) } } // **Every verb that writes the hand-act log is in handActVerbs**, so whether it is a repair is said // where S15 reads it, not left to a default nobody chose. func TestEveryVerbThatRecordsAHandActIsInTheTable(t *testing.T) { listed := map[string]bool{} for _, v := range handActVerbs { if listed[v.Verb] { t.Errorf("%s is in the table twice", v.Verb) } listed[v.Verb] = true if len(v.DecidedFor) > 0 && v.Decision == "" { t.Errorf("%s limits a decision it does not state", v.Verb) } } files, err := filepath.Glob("*.go") if err != nil { t.Fatal(err) } literal := regexp.MustCompile(`(?:\.record\(ctx, |HandAct\{Verb: |RetireApproved: |RetireRejected: |RetireDeleted: )"([^"]+)"\s*[,})]`) composed := regexp.MustCompile(`\.record\(ctx, "([^"]+ )"\s*\+`) found := 0 for _, name := range files { if strings.HasSuffix(name, "_test.go") { continue } body, err := os.ReadFile(name) if err != nil { t.Fatal(err) } for _, m := range literal.FindAllStringSubmatch(string(body), -1) { found++ if !listed[m[1]] { t.Errorf("%s records %q, which handActVerbs does not list", name, m[1]) } } // "plans " + stop|close, "retire " + approve|reject: some listed verb begins with it. for _, m := range composed.FindAllStringSubmatch(string(body), -1) { found++ if !slices.ContainsFunc(handActVerbs, func(v handActVerb) bool { return strings.HasPrefix(v.Verb, m[1]) }) { t.Errorf("%s records %q…, which no verb of handActVerbs begins with", name, m[1]) } } } if found < 12 { t.Fatalf("found %d recording verbs, fewer than the table's own: the search no longer sees them", found) } } // standingSaid is a provider's failing word last said at a moment. func standingSaid(at time.Time) conditions.Condition { return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at} } func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) seen := map[string]bool{} for _, row := range signalsTable { t.Run(row.Row, func(t *testing.T) { if seen[row.Row] { t.Fatalf("%s is in the table twice", row.Row) } seen[row.Row] = true if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" || (row.Severity != conditions.Urgent && row.Severity != conditions.Warning) { t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row) } if row.Deferred != "" { if row.watch != nil || row.Phase <= 1 { t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row) } if _, has := suppressions[row.Row]; has { t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row) } return } if row.watch == nil || row.needs == nil || row.newest == nil { t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row) } s, ok := suppressions[row.Row] if !ok { t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row) } if got := row.watch(calm(now)); len(got) != 0 { t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got) } inside := calm(now) s.inside(inside) if got := row.watch(inside); len(got) != 0 { t.Fatalf("%s raised inside its bound: %+v", row.Row, got) } past := calm(now) s.past(past) got := row.watch(past) if len(got) == 0 { t.Fatalf("%s raised nothing past its bound", row.Row) } for _, o := range got { if !slices.Contains(kindsOf(row), o.Kind) { t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind) } if o.Severity != row.Severity { t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity) } if strings.TrimSpace(o.Summary) == "" { t.Errorf("%s raised a condition that says nothing", row.Row) } } // Through the store: raised past the bound, cleared when the signal returns. store := conditions.NewInMemory() told := &conditions.Told{} k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told, Now: func() time.Time { return now }}) defer k.Close(context.Background()) w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} w.see(t.Context(), past) open, err := k.Open(t.Context()) if err != nil || len(open) != len(got) { t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got)) } w.see(t.Context(), calm(now)) if open, _ := k.Open(t.Context()); len(open) != 0 { t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open) } }) } for name := range suppressions { if !seen[name] { t.Errorf("a suppression for %s, which the table does not have", name) } } } // **The control node silent for half an hour is urgent** (S1); any other machine stays a warning. func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) { now := time.Now() f := calm(now) f.machines[0].lastHeard = now.Add(-31 * time.Minute) f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)}) got := watchHeartbeats(f) if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning { t.Fatalf("%+v", got) } } // **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is. func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) { now := time.Now() f := calm(now) f.machines[0].lastHeard = now.Add(-2 * time.Hour) f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)} if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 { t.Fatalf("a sleeping machine raised %+v", got) } f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)} if got := watchHeartbeats(f); len(got) != 1 { t.Fatalf("a woken machine silent past its bound raised %+v", got) } } // **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store // unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again. func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) { now := time.Now() store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) defer k.Close(context.Background()) w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} silent := calm(now) silent.machines[0].lastHeard = now.Add(-10 * time.Minute) w.see(t.Context(), silent) blind := calm(now) blind.machines, blind.machinesErr = nil, errors.New("the store is away") // Blind for one tick is a read that did not answer in time; for two in a row, a watchdog that cannot // see (novox/hq issue 277). w.see(t.Context(), blind) if open, _ := k.Open(t.Context()); len(open) != 1 { t.Fatalf("one blind tick raised %+v", open) } w.see(t.Context(), blind) open, err := k.Open(t.Context()) if err != nil { t.Fatal(err) } var keys []string for _, c := range open { keys = append(keys, c.Key) } for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} { if !slices.Contains(keys, want) { t.Errorf("%s is not open while the machines cannot be read: %v", want, keys) } } w.see(t.Context(), calm(now)) if open, _ := k.Open(t.Context()); len(open) != 0 { t.Fatalf("seeing again left open %+v", open) } } // **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call // every machine silent. func TestAControllerStandingBySaysNothing(t *testing.T) { store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) defer k.Close(context.Background()) w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }} w.tick(t.Context()) if w.lastTick().IsZero() { t.Fatal("a tick standing by was not counted") } if open, _ := k.Open(t.Context()); len(open) != 0 { t.Fatalf("%+v", open) } } // **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch // its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer. func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) { now := time.Now() f := calm(now) ended := now.Add(-time.Minute) f.staleRefusals = refusedBy(now, 41, 6) f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z", Host: "anchor", Ended: &ended, How: inventory.EpochExpired} got := watchStaleRefusals(f) if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" || !strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") || got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) { t.Fatalf("the stale writer is not named: %+v", got) } // An account the controller refused names the machine whose node-engine sent it. f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6, Receivers: []string{"controller"}, Last: now}} got = watchStaleRefusals(f) if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" { t.Fatalf("a node-engine sending older accounts is not named: %+v", got) } } // **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12). func TestALeaseLostOrMissingIsSaid(t *testing.T) { now := time.Now() f := calm(now) expired := now.Add(-59 * time.Minute) f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor", Ended: &expired, How: inventory.EpochExpired}} got := watchLease(f) if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") || got[0].Severity != conditions.Urgent { t.Fatalf("a holder that stopped renewing was not said: %+v", got) } long := now.Add(-61 * time.Minute) f.lease.ended[0].Ended = &long if got := watchLease(f); len(got) != 0 { t.Fatalf("a loss an hour old is still said: %+v", got) } f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired if got := watchLease(f); len(got) != 0 { t.Fatalf("a lease given back is said as lost: %+v", got) } f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"} if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" { t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got) } f.lease = leaseFacts{unleased: "the bus refused the key"} if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" { t.Fatalf("serving without the lease was not said: %+v", got) } } // **A walk waiting four hours for its delivery's word is urgent** (S16, novox/hq ADR 0239), and says how on. func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) f := calm(now) f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", since: now.Add(-4*time.Hour - time.Minute)}} got := watchWaits(f) if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Key() != "plan.plan-2.waiting" { t.Fatalf("four hours of waiting raised %+v", got) } if !strings.Contains(got[0].Summary, "plans go plan-2") || !strings.Contains(got[0].Summary, "mesh-delivery") { t.Fatalf("it does not say how on: %q", got[0].Summary) } } // **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR // 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised // `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded // before it clear on the next tick, and the cause word alone on any other verb still counts — whether // an act is a drill is said by the verb chosen, not by a word typed into a repair's cause. func TestADrillIsNoRepairAndItsHealerWantedClears(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) f := calm(now) f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act drill", causeDrill), actOf(now.Add(-time.Hour), "hand-act drill", causeDrill), actOf(now.Add(-time.Minute), "hand-act drill", causeDrill)} if got := watchHandActs(f); len(got) != 0 { t.Errorf("drills repeated asked for a healer: %+v", got) } for _, verb := range []string{"push", "conditions silence", "plans close", "a verb nobody listed"} { f := calm(now) f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), verb, causeDrill), actOf(now.Add(-time.Hour), verb, causeDrill)} if got := watchHandActs(f); len(got) != 1 { t.Errorf("%s with the cause %q passed for a drill: %+v", verb, causeDrill, got) } } store := conditions.NewInMemory() k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}, Now: func() time.Time { return now }}) defer k.Close(context.Background()) before := []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "hand-acts." + causeDrill, Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning, Summary: "\"drill\" was repaired by hand 2 times in 14 days"}} if err := k.Reconcile(t.Context(), "S15", before); err != nil { t.Fatal(err) } if open, _ := k.Open(t.Context()); len(open) != 1 { t.Fatalf("the condition of the build before was not open: %+v", open) } f = calm(now) f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", causeDrill), actOf(now.Add(-30*time.Minute), "hand-act record", causeDrill)} w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} w.see(t.Context(), f) if open, _ := k.Open(t.Context()); len(open) != 0 { t.Fatalf("a healer-wanted for two drills stayed open: %+v", open) } }