Archived
jochen
08e11ad761
Approving a retirement and deleting what was retired are a person's act by design (hq ADR 0230); counted by S15 they would ask for a healer that must not exist.
393 lines
17 KiB
Go
393 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
)
|
|
|
|
// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
|
|
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
|
|
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
|
|
// and the condition clears. A row that is not watched says why. A row added to the table without a
|
|
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.
|
|
|
|
// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
|
|
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
|
|
// running, the self-check a minute old.
|
|
func calm(now time.Time) *signalFacts {
|
|
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
|
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
|
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
|
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
|
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
|
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
|
loop: loopFacts{took: now.Add(-time.Second), pending: 1},
|
|
mergesPassed: now.Add(-time.Minute),
|
|
selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
|
|
lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
|
|
lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
|
|
}
|
|
}
|
|
|
|
// refusedBy is n refusals of one writer, the last a moment ago.
|
|
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
|
|
return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
|
|
Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
|
|
}
|
|
|
|
// suppression is one row's signal held back: inside its bound, and past it.
|
|
type suppression struct{ inside, past func(f *signalFacts) }
|
|
|
|
// suppressions are every watched row's, by row.
|
|
var suppressions = map[string]suppression{
|
|
"S1": {
|
|
inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
|
|
past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
|
|
},
|
|
"S2": {
|
|
inside: func(f *signalFacts) {
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
|
|
},
|
|
},
|
|
"S3": {
|
|
inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
|
|
past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
|
|
},
|
|
"S4": {
|
|
inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
|
|
past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
|
|
},
|
|
"S5": {
|
|
inside: func(f *signalFacts) {},
|
|
past: func(f *signalFacts) {
|
|
f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
|
|
},
|
|
},
|
|
"S6": {
|
|
inside: func(f *signalFacts) {
|
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
|
on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
|
|
on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
|
|
},
|
|
},
|
|
"S7": {
|
|
inside: func(f *signalFacts) {
|
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
|
|
},
|
|
},
|
|
"S8": {
|
|
inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
|
|
past: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
|
|
},
|
|
"S9": {
|
|
inside: func(f *signalFacts) {
|
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
|
|
First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
|
|
},
|
|
},
|
|
"S10": {
|
|
inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
|
|
past: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
|
|
},
|
|
"S11": {
|
|
inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
|
|
past: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
|
|
},
|
|
"S12": {
|
|
inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
|
|
past: func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
|
|
},
|
|
"S13": {
|
|
inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
|
|
past: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
|
|
},
|
|
// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
|
|
"S15": {
|
|
inside: func(f *signalFacts) {
|
|
f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"),
|
|
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
|
},
|
|
past: func(f *signalFacts) {
|
|
f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"),
|
|
actByHand(f.now.Add(-time.Hour), "consumer-behind")}
|
|
},
|
|
},
|
|
}
|
|
|
|
// actByHand is one entry in the hand-act log, with its cause.
|
|
func actByHand(at time.Time, cause string) link.HandAct {
|
|
return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop",
|
|
Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause}
|
|
}
|
|
|
|
// **S15 names the cause and, where a healer answers it, that the healer was not enough.**
|
|
func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"),
|
|
actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"),
|
|
actByHand(now.Add(-time.Minute), "restarted the proxy")}
|
|
got := watchHandActs(f)
|
|
if len(got) != 2 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") {
|
|
t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary)
|
|
}
|
|
if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" ||
|
|
!strings.Contains(got[1].Summary, "a healer is wanted") {
|
|
t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary)
|
|
}
|
|
}
|
|
|
|
// **Approving a retirement and deleting what was retired are a person's decision by design** (ADR
|
|
// 0230): repeated, they are not a healer wanted.
|
|
func TestARetirementDecisionRepeatedWantsNoHealer(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
f := calm(now)
|
|
f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), kindRetireWaiting),
|
|
actByHand(now.Add(-time.Hour), kindRetireWaiting), actByHand(now.Add(-time.Hour), kindCleanupWaiting),
|
|
actByHand(now.Add(-time.Minute), kindCleanupWaiting)}
|
|
if got := watchHandActs(f); len(got) != 0 {
|
|
t.Fatalf("a person's decision asked for a healer: %+v", got)
|
|
}
|
|
}
|
|
|
|
// standingSaid is a provider's failing word last said at a moment.
|
|
func standingSaid(at time.Time) conditions.Condition {
|
|
return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
|
|
}
|
|
|
|
func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
|
|
now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
|
|
seen := map[string]bool{}
|
|
for _, row := range signalsTable {
|
|
t.Run(row.Row, func(t *testing.T) {
|
|
if seen[row.Row] {
|
|
t.Fatalf("%s is in the table twice", row.Row)
|
|
}
|
|
seen[row.Row] = true
|
|
if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
|
|
(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
|
|
t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
|
|
}
|
|
if row.Deferred != "" {
|
|
if row.watch != nil || row.Phase <= 1 {
|
|
t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
|
|
}
|
|
if _, has := suppressions[row.Row]; has {
|
|
t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
|
|
}
|
|
return
|
|
}
|
|
if row.watch == nil || row.needs == nil || row.newest == nil {
|
|
t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
|
|
}
|
|
s, ok := suppressions[row.Row]
|
|
if !ok {
|
|
t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
|
|
}
|
|
if got := row.watch(calm(now)); len(got) != 0 {
|
|
t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
|
|
}
|
|
inside := calm(now)
|
|
s.inside(inside)
|
|
if got := row.watch(inside); len(got) != 0 {
|
|
t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
|
|
}
|
|
past := calm(now)
|
|
s.past(past)
|
|
got := row.watch(past)
|
|
if len(got) == 0 {
|
|
t.Fatalf("%s raised nothing past its bound", row.Row)
|
|
}
|
|
for _, o := range got {
|
|
if !slices.Contains(kindsOf(row), o.Kind) {
|
|
t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
|
|
}
|
|
if o.Severity != row.Severity {
|
|
t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
|
|
}
|
|
if strings.TrimSpace(o.Summary) == "" {
|
|
t.Errorf("%s raised a condition that says nothing", row.Row)
|
|
}
|
|
}
|
|
|
|
// Through the store: raised past the bound, cleared when the signal returns.
|
|
store := conditions.NewInMemory()
|
|
told := &conditions.Told{}
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
|
|
Now: func() time.Time { return now }})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
w.see(t.Context(), past)
|
|
open, err := k.Open(t.Context())
|
|
if err != nil || len(open) != len(got) {
|
|
t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
|
|
}
|
|
w.see(t.Context(), calm(now))
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
|
|
}
|
|
})
|
|
}
|
|
for name := range suppressions {
|
|
if !seen[name] {
|
|
t.Errorf("a suppression for %s, which the table does not have", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
|
|
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
f.machines[0].lastHeard = now.Add(-31 * time.Minute)
|
|
f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
|
|
got := watchHeartbeats(f)
|
|
if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
}
|
|
|
|
// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
|
|
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
|
|
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
|
|
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
|
|
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
|
|
t.Fatalf("a sleeping machine raised %+v", got)
|
|
}
|
|
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
|
|
if got := watchHeartbeats(f); len(got) != 1 {
|
|
t.Fatalf("a woken machine silent past its bound raised %+v", got)
|
|
}
|
|
}
|
|
|
|
// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
|
|
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
|
|
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
|
|
now := time.Now()
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
|
silent := calm(now)
|
|
silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
|
|
w.see(t.Context(), silent)
|
|
blind := calm(now)
|
|
blind.machines, blind.machinesErr = nil, errors.New("the store is away")
|
|
w.see(t.Context(), blind)
|
|
open, err := k.Open(t.Context())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var keys []string
|
|
for _, c := range open {
|
|
keys = append(keys, c.Key)
|
|
}
|
|
for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
|
|
if !slices.Contains(keys, want) {
|
|
t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
|
|
}
|
|
}
|
|
w.see(t.Context(), calm(now))
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("seeing again left open %+v", open)
|
|
}
|
|
}
|
|
|
|
// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
|
|
// every machine silent.
|
|
func TestAControllerStandingBySaysNothing(t *testing.T) {
|
|
store := conditions.NewInMemory()
|
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
|
|
defer k.Close(context.Background())
|
|
w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
|
|
w.tick(t.Context())
|
|
if w.lastTick().IsZero() {
|
|
t.Fatal("a tick standing by was not counted")
|
|
}
|
|
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
|
t.Fatalf("%+v", open)
|
|
}
|
|
}
|
|
|
|
// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
|
|
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
|
|
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
ended := now.Add(-time.Minute)
|
|
f.staleRefusals = refusedBy(now, 41, 6)
|
|
f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
|
|
Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
|
|
got := watchStaleRefusals(f)
|
|
if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
|
|
!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
|
|
got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
|
|
t.Fatalf("the stale writer is not named: %+v", got)
|
|
}
|
|
// An account the controller refused names the machine whose node-engine sent it.
|
|
f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
|
|
Receivers: []string{"controller"}, Last: now}}
|
|
got = watchStaleRefusals(f)
|
|
if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
|
|
t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
|
|
}
|
|
}
|
|
|
|
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
|
|
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
|
now := time.Now()
|
|
f := calm(now)
|
|
expired := now.Add(-59 * time.Minute)
|
|
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
|
Ended: &expired, How: inventory.EpochExpired}}
|
|
got := watchLease(f)
|
|
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
|
got[0].Severity != conditions.Urgent {
|
|
t.Fatalf("a holder that stopped renewing was not said: %+v", got)
|
|
}
|
|
long := now.Add(-61 * time.Minute)
|
|
f.lease.ended[0].Ended = &long
|
|
if got := watchLease(f); len(got) != 0 {
|
|
t.Fatalf("a loss an hour old is still said: %+v", got)
|
|
}
|
|
f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
|
|
if got := watchLease(f); len(got) != 0 {
|
|
t.Fatalf("a lease given back is said as lost: %+v", got)
|
|
}
|
|
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
|
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
|
}
|
|
f.lease = leaseFacts{unleased: "the bus refused the key"}
|
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
|
|
t.Fatalf("serving without the lease was not said: %+v", got)
|
|
}
|
|
}
|