jschoubben
f83fcdc15f
The dead-letter row and a max-deliveries advisory without a token both said bus.<stream>.<consumer>.max-deliveries: each look added an observation and a line of evidence through the row, and the two overwrote each other's words. The row owns the key since issue 330, so the advisory watcher leaves it, and the row now says when the newest held message was given up, so a letter held for a day no longer reads as observed every 30 seconds. Times without Happened is refused, since the raise ignored it and an update counted it.
633 lines
28 KiB
Go
633 lines
28 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
|
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
|
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
|
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
|
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
|
|
|
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
|
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
|
// running, the self-check a minute old.
|
|
func calm(now time.Time) *signalFacts {
|
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
|
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
|
mergesPassed: now.Add(-time.Minute),
|
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
|
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
|
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
|
facts: factsFacts{taken: now.Add(-time.Hour), began: now.Add(-time.Hour)},
|
|
}
|
|
}
|
|
|
|
// refusedBy is n refusals of one writer, the last a moment ago.
|
|
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
|
|
return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
|
|
Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
|
|
}
|
|
|
|
// suppression is one row's signal held back: inside its bound, and past it.
|
|
type suppression struct{ inside, past func(f *signalFacts) }
|
|
|
|
// suppressions are every watched row's, by row.
|
|
var suppressions = map[string]suppression{
|
|
"S1": {
|
|
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
|
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
|
},
|
|
"S2": {
|
|
inside: func(f *signalFacts) {
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
|
},
|
|
},
|
|
"S3": {
|
|
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
|
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
|
},
|
|
"S4": {
|
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
|
},
|
|
"S5": {
|
|
inside: func(f *signalFacts) {},
|
|
past: func(f *signalFacts) {
|
|
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
|
},
|
|
},
|
|
"S6": {
|
|
inside: func(f *signalFacts) {
|
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
|
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
|
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
|
},
|
|
},
|
|
"S7": {
|
|
inside: func(f *signalFacts) {
|
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
|
},
|
|
},
|
|
"S8": {
|
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
|
},
|
|
// A message given up on and not kept: the one max-deliveries advisory S9 says itself (issue 440).
|
|
"S9": {
|
|
inside: func(f *signalFacts) {
|
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
|
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop",
|
|
Token: link.AdvisoryNotKept, Said: "gave up, not kept", First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
|
},
|
|
},
|
|
"S10": {
|
|
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
|
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
|
},
|
|
"S11": {
|
|
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
|
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
|
},
|
|
"S12": {
|
|
inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
|
|
past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
|
|
},
|
|
"S13": {
|
|
inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
|
|
past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
|
|
},
|
|
// The snapshot a merge check is fed, older than two days; or none kept by a controller two days up.
|
|
"S14": {
|
|
inside: func(f *signalFacts) { f.facts.taken = f.now.Add(-47 * time.Hour) },
|
|
past: func(f *signalFacts) { f.facts.taken = f.now.Add(-49 * time.Hour) },
|
|
},
|
|
// A walk waiting for its delivery's word for 30 minutes (novox/hq ADR 0239).
|
|
"S16": {
|
|
inside: func(f *signalFacts) {
|
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
|
|
since: f.now.Add(-29 * time.Minute)}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
|
|
since: f.now.Add(-31 * time.Minute)}}
|
|
},
|
|
},
|
|
// A batch still assembling past its maximum with no walk open (novox/hq ADR 0276): a minute's grace.
|
|
"S18": {
|
|
inside: func(f *signalFacts) {
|
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11",
|
|
atMost: f.now.Add(-59 * time.Second), closed: f.now.Add(-59 * time.Second)}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11",
|
|
atMost: f.now.Add(-61 * time.Second), closed: f.now.Add(-61 * time.Second)}}
|
|
},
|
|
},
|
|
// A batch queued behind an open walk past that walk's bound, named.
|
|
"S19": {
|
|
inside: func(f *signalFacts) {
|
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11",
|
|
closed: f.now.Add(-59 * time.Minute), behind: "plan-1", walkBound: time.Hour}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11",
|
|
closed: f.now.Add(-61 * time.Minute), behind: "plan-1", walkBound: time.Hour}}
|
|
},
|
|
},
|
|
// A send refused because it would replace the bus outside its planned step: said at its first refusal,
|
|
// whatever the bound (novox/hq issue 336). Inside: a new bus build waits, and no send was refused for it.
|
|
"S17": {
|
|
inside: func(f *signalFacts) {
|
|
f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"},
|
|
machines: []string{"anchor"}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"},
|
|
machines: []string{"anchor"}, waits: []busWaitFacts{{plan: "plan-1", repository: "novox/app",
|
|
commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}}
|
|
},
|
|
},
|
|
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
|
"S15": {
|
|
inside: func(f *signalFacts) {
|
|
f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"),
|
|
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"),
|
|
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
|
},
|
|
},
|
|
}
|
|
|
|
// actByHand is one entry in the hand-act log, with its cause.
|
|
func actByHand(at time.Time, cause string) link.HandAct {
|
|
return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop",
|
|
Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause}
|
|
}
|
|
|
|
// **S15 names the cause and, where a healer answers it, that the healer was not enough.**
|
|
func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"),
|
|
actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"),
|
|
actByHand(now.Add(-time.Minute), "restarted the proxy")}
|
|
got := watchHandActs(f)
|
|
if len(got) != 2 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") {
|
|
t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary)
|
|
}
|
|
if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" ||
|
|
!strings.Contains(got[1].Summary, "a healer is wanted") {
|
|
t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary)
|
|
}
|
|
}
|
|
|
|
// actOf is an act in the log recorded by a verb, with its cause.
|
|
func actOf(at time.Time, verb, cause string) link.HandAct {
|
|
a := actByHand(at, cause)
|
|
a.Verb, a.Args = verb, nil
|
|
return a
|
|
}
|
|
|
|
// **An act a person decides by design is no repair** (handActVerbs): approving or rejecting a
|
|
// retirement and deleting what was retired (ADR 0230), a bus upgrade and a backlog released (ADR 0236),
|
|
// and a rotation after a leak — repeated, none is a healer wanted, whatever cause it gives.
|
|
func TestAPersonsDecisionRepeatedWantsNoHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
for _, c := range []struct{ verb, cause string }{
|
|
{"retire approve", kindRetireWaiting}, {"retire reject", kindRetireWaiting},
|
|
{"cleanup delete", kindCleanupWaiting}, {"bus upgrade", "bus-upgrade"},
|
|
{"bus upgrade", "a word the person chose"}, {"upgrade release-backlog", "upgrade release-backlog"},
|
|
{"secret rotate", causeLeakedInLogs},
|
|
} {
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause),
|
|
actOf(now.Add(-time.Hour), c.verb, c.cause)}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Errorf("%s (cause %s) repeated asked for a healer: %+v", c.verb, c.cause, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **What repairs still counts**: a push by hand above all (ADR 0236 exists to end it), a rotation for
|
|
// any cause but a leak, an act recorded outside the mesh whatever cause it names, and a verb the table
|
|
// does not know.
|
|
func TestARepairRepeatedStillWantsAHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
for _, c := range []struct{ verb, cause string }{
|
|
{"push", "push"}, {"plans close", "plans close"}, {"conditions silence", "consumer-behind"},
|
|
{"secret rotate", "stopped-working"}, {"hand-act record", "bus-upgrade"},
|
|
{"hand-act record", kindCleanupWaiting}, {"a verb nobody listed", "x"},
|
|
} {
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause),
|
|
actOf(now.Add(-time.Hour), c.verb, c.cause)}
|
|
if got := watchHandActs(f); len(got) != 1 || got[0].Kind != "healer-wanted" {
|
|
t.Errorf("%s (cause %s) repeated wanted no healer: %+v", c.verb, c.cause, got)
|
|
}
|
|
}
|
|
// A decision does not make up the second of a cause a repair recorded once.
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act record", "bus-upgrade"),
|
|
actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Errorf("one repair and one decision asked for a healer: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A healer-wanted already open for a decision clears on the next tick** — the log of 2026-10-06:
|
|
// a bus upgrade recorded after the fact and one through its verb, and two rotations after a leak.
|
|
func TestAHealerWantedOpenForADecisionClearsOnTheNextTick(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
// What the build before this one raised, as it raised it.
|
|
var before []conditions.Observation
|
|
for _, cause := range []string{"bus-upgrade", causeLeakedInLogs} {
|
|
before = append(before, conditions.Observation{Scope: conditions.ScopeMesh, ID: "hand-acts." + cause,
|
|
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning, Summary: "repaired twice"})
|
|
}
|
|
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 2 {
|
|
t.Fatalf("the conditions of the build before were not open: %+v", open)
|
|
}
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", "bus-upgrade"),
|
|
actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs),
|
|
actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs),
|
|
actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")}
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), f)
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("a healer-wanted for a person's decision stayed open: %+v", open)
|
|
}
|
|
}
|
|
|
|
// **Every verb that writes the hand-act log is in handActVerbs**, so whether it is a repair is said
|
|
// where S15 reads it, not left to a default nobody chose.
|
|
func TestEveryVerbThatRecordsAHandActIsInTheTable(t *testing.T) {
|
|
listed := map[string]bool{}
|
|
for _, v := range handActVerbs {
|
|
if listed[v.Verb] {
|
|
t.Errorf("%s is in the table twice", v.Verb)
|
|
}
|
|
listed[v.Verb] = true
|
|
if len(v.DecidedFor) > 0 && v.Decision == "" {
|
|
t.Errorf("%s limits a decision it does not state", v.Verb)
|
|
}
|
|
}
|
|
files, err := filepath.Glob("*.go")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
literal := regexp.MustCompile(`(?:\.record\(ctx, |HandAct\{Verb: |RetireApproved: |RetireRejected: |RetireDeleted: )"([^"]+)"\s*[,})]`)
|
|
composed := regexp.MustCompile(`\.record\(ctx, "([^"]+ )"\s*\+`)
|
|
found := 0
|
|
for _, name := range files {
|
|
if strings.HasSuffix(name, "_test.go") {
|
|
continue
|
|
}
|
|
body, err := os.ReadFile(name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, m := range literal.FindAllStringSubmatch(string(body), -1) {
|
|
found++
|
|
if !listed[m[1]] {
|
|
t.Errorf("%s records %q, which handActVerbs does not list", name, m[1])
|
|
}
|
|
}
|
|
// "plans " + stop|close, "retire " + approve|reject: some listed verb begins with it.
|
|
for _, m := range composed.FindAllStringSubmatch(string(body), -1) {
|
|
found++
|
|
if !slices.ContainsFunc(handActVerbs, func(v handActVerb) bool { return strings.HasPrefix(v.Verb, m[1]) }) {
|
|
t.Errorf("%s records %q…, which no verb of handActVerbs begins with", name, m[1])
|
|
}
|
|
}
|
|
}
|
|
if found < 12 {
|
|
t.Fatalf("found %d recording verbs, fewer than the table's own: the search no longer sees them", found)
|
|
}
|
|
}
|
|
|
|
// standingSaid is a provider's failing word last said at a moment.
|
|
func standingSaid(at time.Time) conditions.Condition {
|
|
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
|
}
|
|
|
|
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
seen := map[string]bool{}
|
|
for _, row := range signalsTable {
|
|
t.Run(row.Row, func(t *testing.T) {
|
|
if seen[row.Row] {
|
|
t.Fatalf("%s is in the table twice", row.Row)
|
|
}
|
|
seen[row.Row] = true
|
|
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
|
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
|
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
|
}
|
|
if row.Deferred != "" {
|
|
if row.watch != nil || row.Phase <= 1 {
|
|
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
|
}
|
|
if _, has := suppressions[row.Row]; has {
|
|
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
|
}
|
|
return
|
|
}
|
|
if row.watch == nil || row.needs == nil || row.newest == nil {
|
|
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
|
}
|
|
s, ok := suppressions[row.Row]
|
|
if !ok {
|
|
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
|
}
|
|
if got := row.watch(calm(now)); len(got) != 0 {
|
|
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
|
}
|
|
inside := calm(now)
|
|
s.inside(inside)
|
|
if got := row.watch(inside); len(got) != 0 {
|
|
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
|
}
|
|
past := calm(now)
|
|
s.past(past)
|
|
got := row.watch(past)
|
|
if len(got) == 0 {
|
|
t.Fatalf("%s raised nothing past its bound", row.Row)
|
|
}
|
|
for _, o := range got {
|
|
if !slices.Contains(kindsOf(row), o.Kind) {
|
|
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
|
}
|
|
if o.Severity != row.Severity {
|
|
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
|
}
|
|
if strings.TrimSpace(o.Summary) == "" {
|
|
t.Errorf("%s raised a condition that says nothing", row.Row)
|
|
}
|
|
}
|
|
|
|
// Through the store: raised past the bound, cleared when the signal returns.
|
|
store := conditions.NewInMemory()
|
|
told := &conditions.Told{}
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
|
Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), past)
|
|
open, err := k.Open(t.Context())
|
|
if err != nil || len(open) != len(got) {
|
|
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
|
}
|
|
w.see(t.Context(), calm(now))
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
|
}
|
|
})
|
|
}
|
|
for name := range suppressions {
|
|
if !seen[name] {
|
|
t.Errorf("a suppression for %s, which the table does not have", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
|
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
|
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
|
got := watchHeartbeats(f)
|
|
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
}
|
|
|
|
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
|
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
|
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
|
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
|
t.Fatalf("a sleeping machine raised %+v", got)
|
|
}
|
|
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
|
if got := watchHeartbeats(f); len(got) != 1 {
|
|
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
|
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
|
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
|
now := time.Now()
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
silent := calm(now)
|
|
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
|
w.see(t.Context(), silent)
|
|
blind := calm(now)
|
|
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
|
// Blind for one tick is a read that did not answer in time; for two in a row, a watchdog that cannot
|
|
// see (novox/hq issue 277).
|
|
w.see(t.Context(), blind)
|
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
|
t.Fatalf("one blind tick raised %+v", open)
|
|
}
|
|
w.see(t.Context(), blind)
|
|
open, err := k.Open(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var keys []string
|
|
for _, c := range open {
|
|
keys = append(keys, c.Key)
|
|
}
|
|
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
|
if !slices.Contains(keys, want) {
|
|
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
|
}
|
|
}
|
|
w.see(t.Context(), calm(now))
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("seeing again left open %+v", open)
|
|
}
|
|
}
|
|
|
|
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
|
// every machine silent.
|
|
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
|
w.tick(t.Context())
|
|
if w.lastTick().IsZero() {
|
|
t.Fatal("a tick standing by was not counted")
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("%+v", open)
|
|
}
|
|
}
|
|
|
|
// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
|
|
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
|
|
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
ended := now.Add(-time.Minute)
|
|
f.staleRefusals = refusedBy(now, 41, 6)
|
|
f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
|
|
Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
|
|
got := watchStaleRefusals(f)
|
|
if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
|
|
!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
|
|
got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
|
|
t.Fatalf("the stale writer is not named: %+v", got)
|
|
}
|
|
// An account the controller refused names the machine whose node-engine sent it.
|
|
f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
|
|
Receivers: []string{"controller"}, Last: now}}
|
|
got = watchStaleRefusals(f)
|
|
if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
|
|
t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
|
|
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
expired := now.Add(-59 * time.Minute)
|
|
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
|
Ended: &expired, How: inventory.EpochExpired}}
|
|
got := watchLease(f)
|
|
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
|
got[0].Severity != conditions.Urgent {
|
|
t.Fatalf("a holder that stopped renewing was not said: %+v", got)
|
|
}
|
|
long := now.Add(-61 * time.Minute)
|
|
f.lease.ended[0].Ended = &long
|
|
if got := watchLease(f); len(got) != 0 {
|
|
t.Fatalf("a loss an hour old is still said: %+v", got)
|
|
}
|
|
f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
|
|
if got := watchLease(f); len(got) != 0 {
|
|
t.Fatalf("a lease given back is said as lost: %+v", got)
|
|
}
|
|
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
|
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
|
}
|
|
f.lease = leaseFacts{unleased: "the bus refused the key"}
|
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
|
|
t.Fatalf("serving without the lease was not said: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A walk waiting four hours for its delivery's word is urgent** (S16, novox/hq ADR 0239), and says how on.
|
|
func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
|
|
since: now.Add(-4*time.Hour - time.Minute)}}
|
|
got := watchWaits(f)
|
|
if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Key() != "plan.plan-2.waiting" {
|
|
t.Fatalf("four hours of waiting raised %+v", got)
|
|
}
|
|
if !strings.Contains(got[0].Summary, "plans go plan-2") || !strings.Contains(got[0].Summary, "mesh-delivery") {
|
|
t.Fatalf("it does not say how on: %q", got[0].Summary)
|
|
}
|
|
}
|
|
|
|
// **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR
|
|
// 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised
|
|
// `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded
|
|
// before it clear on the next tick, and the cause word alone on any other verb still counts — whether
|
|
// an act is a drill is said by the verb chosen, not by a word typed into a repair's cause.
|
|
func TestADrillIsNoRepairAndItsHealerWantedClears(t *testing.T) {
|
|
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act drill", causeDrill),
|
|
actOf(now.Add(-time.Hour), "hand-act drill", causeDrill), actOf(now.Add(-time.Minute), "hand-act drill", causeDrill)}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Errorf("drills repeated asked for a healer: %+v", got)
|
|
}
|
|
for _, verb := range []string{"push", "conditions silence", "plans close", "a verb nobody listed"} {
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), verb, causeDrill), actOf(now.Add(-time.Hour), verb, causeDrill)}
|
|
if got := watchHandActs(f); len(got) != 1 {
|
|
t.Errorf("%s with the cause %q passed for a drill: %+v", verb, causeDrill, got)
|
|
}
|
|
}
|
|
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
before := []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "hand-acts." + causeDrill,
|
|
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
|
Summary: "\"drill\" was repaired by hand 2 times in 14 days"}}
|
|
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
|
t.Fatalf("the condition of the build before was not open: %+v", open)
|
|
}
|
|
f = calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", causeDrill),
|
|
actOf(now.Add(-30*time.Minute), "hand-act record", causeDrill)}
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), f)
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("a healer-wanted for two drills stayed open: %+v", open)
|
|
}
|
|
}
|