jochen
2799e95035
Two ADR 0240 drills recorded with hand-act record --cause drill raised mesh.hand-acts.drill.healer-wanted. hand-act drill (seat verb drill) records them as a person's decision; hand-act record now refuses the cause, so the two recorded before it clear on the next tick and a repair cannot pass for a drill by the word it gives.
597 lines
26 KiB
Go
597 lines
26 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
|
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
|
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
|
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
|
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
|
|
|
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
|
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
|
// running, the self-check a minute old.
|
|
func calm(now time.Time) *signalFacts {
|
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
|
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
|
mergesPassed: now.Add(-time.Minute),
|
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
|
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
|
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
|
facts: factsFacts{taken: now.Add(-time.Hour), began: now.Add(-time.Hour)},
|
|
}
|
|
}
|
|
|
|
// refusedBy is n refusals of one writer, the last a moment ago.
|
|
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
|
|
return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
|
|
Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
|
|
}
|
|
|
|
// suppression is one row's signal held back: inside its bound, and past it.
|
|
type suppression struct{ inside, past func(f *signalFacts) }
|
|
|
|
// suppressions are every watched row's, by row.
|
|
var suppressions = map[string]suppression{
|
|
"S1": {
|
|
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
|
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
|
},
|
|
"S2": {
|
|
inside: func(f *signalFacts) {
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
|
},
|
|
},
|
|
"S3": {
|
|
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
|
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
|
},
|
|
"S4": {
|
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
|
},
|
|
"S5": {
|
|
inside: func(f *signalFacts) {},
|
|
past: func(f *signalFacts) {
|
|
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
|
},
|
|
},
|
|
"S6": {
|
|
inside: func(f *signalFacts) {
|
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
|
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
|
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
|
},
|
|
},
|
|
"S7": {
|
|
inside: func(f *signalFacts) {
|
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
|
},
|
|
},
|
|
"S8": {
|
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
|
},
|
|
"S9": {
|
|
inside: func(f *signalFacts) {
|
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
|
},
|
|
},
|
|
"S10": {
|
|
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
|
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
|
},
|
|
"S11": {
|
|
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
|
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
|
},
|
|
"S12": {
|
|
inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
|
|
past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
|
|
},
|
|
"S13": {
|
|
inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
|
|
past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
|
|
},
|
|
// The snapshot a merge check is fed, older than two days; or none kept by a controller two days up.
|
|
"S14": {
|
|
inside: func(f *signalFacts) { f.facts.taken = f.now.Add(-47 * time.Hour) },
|
|
past: func(f *signalFacts) { f.facts.taken = f.now.Add(-49 * time.Hour) },
|
|
},
|
|
// A walk waiting for its delivery's word for 30 minutes (novox/hq ADR 0239).
|
|
"S16": {
|
|
inside: func(f *signalFacts) {
|
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
|
|
since: f.now.Add(-29 * time.Minute)}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
|
|
since: f.now.Add(-31 * time.Minute)}}
|
|
},
|
|
},
|
|
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
|
"S15": {
|
|
inside: func(f *signalFacts) {
|
|
f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"),
|
|
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"),
|
|
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
|
},
|
|
},
|
|
}
|
|
|
|
// actByHand is one entry in the hand-act log, with its cause.
|
|
func actByHand(at time.Time, cause string) link.HandAct {
|
|
return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop",
|
|
Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause}
|
|
}
|
|
|
|
// **S15 names the cause and, where a healer answers it, that the healer was not enough.**
|
|
func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"),
|
|
actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"),
|
|
actByHand(now.Add(-time.Minute), "restarted the proxy")}
|
|
got := watchHandActs(f)
|
|
if len(got) != 2 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") {
|
|
t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary)
|
|
}
|
|
if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" ||
|
|
!strings.Contains(got[1].Summary, "a healer is wanted") {
|
|
t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary)
|
|
}
|
|
}
|
|
|
|
// actOf is an act in the log recorded by a verb, with its cause.
|
|
func actOf(at time.Time, verb, cause string) link.HandAct {
|
|
a := actByHand(at, cause)
|
|
a.Verb, a.Args = verb, nil
|
|
return a
|
|
}
|
|
|
|
// **An act a person decides by design is no repair** (handActVerbs): approving or rejecting a
|
|
// retirement and deleting what was retired (ADR 0230), a bus upgrade and a backlog released (ADR 0236),
|
|
// and a rotation after a leak — repeated, none is a healer wanted, whatever cause it gives.
|
|
func TestAPersonsDecisionRepeatedWantsNoHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
for _, c := range []struct{ verb, cause string }{
|
|
{"retire approve", kindRetireWaiting}, {"retire reject", kindRetireWaiting},
|
|
{"cleanup delete", kindCleanupWaiting}, {"bus upgrade", "bus-upgrade"},
|
|
{"bus upgrade", "a word the person chose"}, {"upgrade release-backlog", "upgrade release-backlog"},
|
|
{"secret rotate", causeLeakedInLogs},
|
|
} {
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause),
|
|
actOf(now.Add(-time.Hour), c.verb, c.cause)}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Errorf("%s (cause %s) repeated asked for a healer: %+v", c.verb, c.cause, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **What repairs still counts**: a push by hand above all (ADR 0236 exists to end it), a rotation for
|
|
// any cause but a leak, an act recorded outside the mesh whatever cause it names, and a verb the table
|
|
// does not know.
|
|
func TestARepairRepeatedStillWantsAHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
for _, c := range []struct{ verb, cause string }{
|
|
{"push", "push"}, {"plans close", "plans close"}, {"conditions silence", "consumer-behind"},
|
|
{"secret rotate", "stopped-working"}, {"hand-act record", "bus-upgrade"},
|
|
{"hand-act record", kindCleanupWaiting}, {"a verb nobody listed", "x"},
|
|
} {
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), c.verb, c.cause),
|
|
actOf(now.Add(-time.Hour), c.verb, c.cause)}
|
|
if got := watchHandActs(f); len(got) != 1 || got[0].Kind != "healer-wanted" {
|
|
t.Errorf("%s (cause %s) repeated wanted no healer: %+v", c.verb, c.cause, got)
|
|
}
|
|
}
|
|
// A decision does not make up the second of a cause a repair recorded once.
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act record", "bus-upgrade"),
|
|
actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Errorf("one repair and one decision asked for a healer: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A healer-wanted already open for a decision clears on the next tick** — the log of 2026-10-06:
|
|
// a bus upgrade recorded after the fact and one through its verb, and two rotations after a leak.
|
|
func TestAHealerWantedOpenForADecisionClearsOnTheNextTick(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC)
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
// What the build before this one raised, as it raised it.
|
|
var before []conditions.Observation
|
|
for _, cause := range []string{"bus-upgrade", causeLeakedInLogs} {
|
|
before = append(before, conditions.Observation{Scope: conditions.ScopeMesh, ID: "hand-acts." + cause,
|
|
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning, Summary: "repaired twice"})
|
|
}
|
|
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 2 {
|
|
t.Fatalf("the conditions of the build before were not open: %+v", open)
|
|
}
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", "bus-upgrade"),
|
|
actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs),
|
|
actOf(now.Add(-8*time.Hour), "secret rotate", causeLeakedInLogs),
|
|
actOf(now.Add(-time.Hour), "bus upgrade", "bus-upgrade")}
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), f)
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("a healer-wanted for a person's decision stayed open: %+v", open)
|
|
}
|
|
}
|
|
|
|
// **Every verb that writes the hand-act log is in handActVerbs**, so whether it is a repair is said
|
|
// where S15 reads it, not left to a default nobody chose.
|
|
func TestEveryVerbThatRecordsAHandActIsInTheTable(t *testing.T) {
|
|
listed := map[string]bool{}
|
|
for _, v := range handActVerbs {
|
|
if listed[v.Verb] {
|
|
t.Errorf("%s is in the table twice", v.Verb)
|
|
}
|
|
listed[v.Verb] = true
|
|
if len(v.DecidedFor) > 0 && v.Decision == "" {
|
|
t.Errorf("%s limits a decision it does not state", v.Verb)
|
|
}
|
|
}
|
|
files, err := filepath.Glob("*.go")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
literal := regexp.MustCompile(`(?:\.record\(ctx, |HandAct\{Verb: |RetireApproved: |RetireRejected: |RetireDeleted: )"([^"]+)"\s*[,})]`)
|
|
composed := regexp.MustCompile(`\.record\(ctx, "([^"]+ )"\s*\+`)
|
|
found := 0
|
|
for _, name := range files {
|
|
if strings.HasSuffix(name, "_test.go") {
|
|
continue
|
|
}
|
|
body, err := os.ReadFile(name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, m := range literal.FindAllStringSubmatch(string(body), -1) {
|
|
found++
|
|
if !listed[m[1]] {
|
|
t.Errorf("%s records %q, which handActVerbs does not list", name, m[1])
|
|
}
|
|
}
|
|
// "plans " + stop|close, "retire " + approve|reject: some listed verb begins with it.
|
|
for _, m := range composed.FindAllStringSubmatch(string(body), -1) {
|
|
found++
|
|
if !slices.ContainsFunc(handActVerbs, func(v handActVerb) bool { return strings.HasPrefix(v.Verb, m[1]) }) {
|
|
t.Errorf("%s records %q…, which no verb of handActVerbs begins with", name, m[1])
|
|
}
|
|
}
|
|
}
|
|
if found < 12 {
|
|
t.Fatalf("found %d recording verbs, fewer than the table's own: the search no longer sees them", found)
|
|
}
|
|
}
|
|
|
|
// standingSaid is a provider's failing word last said at a moment.
|
|
func standingSaid(at time.Time) conditions.Condition {
|
|
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
|
}
|
|
|
|
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
seen := map[string]bool{}
|
|
for _, row := range signalsTable {
|
|
t.Run(row.Row, func(t *testing.T) {
|
|
if seen[row.Row] {
|
|
t.Fatalf("%s is in the table twice", row.Row)
|
|
}
|
|
seen[row.Row] = true
|
|
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
|
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
|
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
|
}
|
|
if row.Deferred != "" {
|
|
if row.watch != nil || row.Phase <= 1 {
|
|
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
|
}
|
|
if _, has := suppressions[row.Row]; has {
|
|
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
|
}
|
|
return
|
|
}
|
|
if row.watch == nil || row.needs == nil || row.newest == nil {
|
|
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
|
}
|
|
s, ok := suppressions[row.Row]
|
|
if !ok {
|
|
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
|
}
|
|
if got := row.watch(calm(now)); len(got) != 0 {
|
|
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
|
}
|
|
inside := calm(now)
|
|
s.inside(inside)
|
|
if got := row.watch(inside); len(got) != 0 {
|
|
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
|
}
|
|
past := calm(now)
|
|
s.past(past)
|
|
got := row.watch(past)
|
|
if len(got) == 0 {
|
|
t.Fatalf("%s raised nothing past its bound", row.Row)
|
|
}
|
|
for _, o := range got {
|
|
if !slices.Contains(kindsOf(row), o.Kind) {
|
|
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
|
}
|
|
if o.Severity != row.Severity {
|
|
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
|
}
|
|
if strings.TrimSpace(o.Summary) == "" {
|
|
t.Errorf("%s raised a condition that says nothing", row.Row)
|
|
}
|
|
}
|
|
|
|
// Through the store: raised past the bound, cleared when the signal returns.
|
|
store := conditions.NewInMemory()
|
|
told := &conditions.Told{}
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
|
Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), past)
|
|
open, err := k.Open(t.Context())
|
|
if err != nil || len(open) != len(got) {
|
|
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
|
}
|
|
w.see(t.Context(), calm(now))
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
|
}
|
|
})
|
|
}
|
|
for name := range suppressions {
|
|
if !seen[name] {
|
|
t.Errorf("a suppression for %s, which the table does not have", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
|
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
|
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
|
got := watchHeartbeats(f)
|
|
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
}
|
|
|
|
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
|
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
|
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
|
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
|
t.Fatalf("a sleeping machine raised %+v", got)
|
|
}
|
|
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
|
if got := watchHeartbeats(f); len(got) != 1 {
|
|
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
|
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
|
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
|
now := time.Now()
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
silent := calm(now)
|
|
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
|
w.see(t.Context(), silent)
|
|
blind := calm(now)
|
|
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
|
// Blind for one tick is a read that did not answer in time; for two in a row, a watchdog that cannot
|
|
// see (novox/hq issue 277).
|
|
w.see(t.Context(), blind)
|
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
|
t.Fatalf("one blind tick raised %+v", open)
|
|
}
|
|
w.see(t.Context(), blind)
|
|
open, err := k.Open(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var keys []string
|
|
for _, c := range open {
|
|
keys = append(keys, c.Key)
|
|
}
|
|
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
|
if !slices.Contains(keys, want) {
|
|
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
|
}
|
|
}
|
|
w.see(t.Context(), calm(now))
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("seeing again left open %+v", open)
|
|
}
|
|
}
|
|
|
|
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
|
// every machine silent.
|
|
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
|
w.tick(t.Context())
|
|
if w.lastTick().IsZero() {
|
|
t.Fatal("a tick standing by was not counted")
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("%+v", open)
|
|
}
|
|
}
|
|
|
|
// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
|
|
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
|
|
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
ended := now.Add(-time.Minute)
|
|
f.staleRefusals = refusedBy(now, 41, 6)
|
|
f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
|
|
Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
|
|
got := watchStaleRefusals(f)
|
|
if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
|
|
!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
|
|
got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
|
|
t.Fatalf("the stale writer is not named: %+v", got)
|
|
}
|
|
// An account the controller refused names the machine whose node-engine sent it.
|
|
f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
|
|
Receivers: []string{"controller"}, Last: now}}
|
|
got = watchStaleRefusals(f)
|
|
if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
|
|
t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
|
|
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
expired := now.Add(-59 * time.Minute)
|
|
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
|
Ended: &expired, How: inventory.EpochExpired}}
|
|
got := watchLease(f)
|
|
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
|
got[0].Severity != conditions.Urgent {
|
|
t.Fatalf("a holder that stopped renewing was not said: %+v", got)
|
|
}
|
|
long := now.Add(-61 * time.Minute)
|
|
f.lease.ended[0].Ended = &long
|
|
if got := watchLease(f); len(got) != 0 {
|
|
t.Fatalf("a loss an hour old is still said: %+v", got)
|
|
}
|
|
f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
|
|
if got := watchLease(f); len(got) != 0 {
|
|
t.Fatalf("a lease given back is said as lost: %+v", got)
|
|
}
|
|
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
|
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
|
}
|
|
f.lease = leaseFacts{unleased: "the bus refused the key"}
|
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
|
|
t.Fatalf("serving without the lease was not said: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A walk waiting four hours for its delivery's word is urgent** (S16, novox/hq ADR 0239), and says how on.
|
|
func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery",
|
|
since: now.Add(-4*time.Hour - time.Minute)}}
|
|
got := watchWaits(f)
|
|
if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Key() != "plan.plan-2.waiting" {
|
|
t.Fatalf("four hours of waiting raised %+v", got)
|
|
}
|
|
if !strings.Contains(got[0].Summary, "plans go plan-2") || !strings.Contains(got[0].Summary, "mesh-delivery") {
|
|
t.Fatalf("it does not say how on: %q", got[0].Summary)
|
|
}
|
|
}
|
|
|
|
// **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR
|
|
// 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised
|
|
// `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded
|
|
// before it clear on the next tick, and the cause word alone on any other verb still counts — whether
|
|
// an act is a drill is said by the verb chosen, not by a word typed into a repair's cause.
|
|
func TestADrillIsNoRepairAndItsHealerWantedClears(t *testing.T) {
|
|
now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), "hand-act drill", causeDrill),
|
|
actOf(now.Add(-time.Hour), "hand-act drill", causeDrill), actOf(now.Add(-time.Minute), "hand-act drill", causeDrill)}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Errorf("drills repeated asked for a healer: %+v", got)
|
|
}
|
|
for _, verb := range []string{"push", "conditions silence", "plans close", "a verb nobody listed"} {
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-26*time.Hour), verb, causeDrill), actOf(now.Add(-time.Hour), verb, causeDrill)}
|
|
if got := watchHandActs(f); len(got) != 1 {
|
|
t.Errorf("%s with the cause %q passed for a drill: %+v", verb, causeDrill, got)
|
|
}
|
|
}
|
|
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
|
Teller: &conditions.Told{}, Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
before := []conditions.Observation{{Scope: conditions.ScopeMesh, ID: "hand-acts." + causeDrill,
|
|
Token: "healer-wanted", Kind: "healer-wanted", Severity: conditions.Warning,
|
|
Summary: "\"drill\" was repaired by hand 2 times in 14 days"}}
|
|
if err := k.Reconcile(t.Context(), "S15", before); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 1 {
|
|
t.Fatalf("the condition of the build before was not open: %+v", open)
|
|
}
|
|
f = calm(now)
|
|
f.handActs = []link.HandAct{actOf(now.Add(-11*time.Hour), "hand-act record", causeDrill),
|
|
actOf(now.Add(-30*time.Minute), "hand-act record", causeDrill)}
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), f)
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("a healer-wanted for two drills stayed open: %+v", open)
|
|
}
|
|
}
|