mesh-controller/cmd/mesh-controller/signals_test.go

package main

import (
	"context"
	"errors"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
// and the condition clears. A row that is not watched says why. A row added to the table without a
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.

// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
// running, the self-check a minute old.
func calm(now time.Time) *signalFacts {
	return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
		machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
			every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
			lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
		plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
			entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
		loop:          loopFacts{took: now.Add(-time.Second), pending: 1},
		mergesPassed:  now.Add(-time.Minute),
		selfCheck:     selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
		lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{},
		lease: leaseFacts{held: true, epoch: 57, renewed: now.Add(-2 * time.Second)},
	}
}

// refusedBy is n refusals of one writer, the last a moment ago.
func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals {
	return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n,
		Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}}
}

// suppression is one row's signal held back: inside its bound, and past it.
type suppression struct{ inside, past func(f *signalFacts) }

// suppressions are every watched row's, by row.
var suppressions = map[string]suppression{
	"S1": {
		inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
		past:   func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
	},
	"S2": {
		inside: func(f *signalFacts) {
			f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
		},
		past: func(f *signalFacts) {
			f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
		},
	},
	"S3": {
		inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
		past:   func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
	},
	"S4": {
		inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
		past:   func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
	},
	"S5": {
		inside: func(f *signalFacts) {},
		past: func(f *signalFacts) {
			f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
		},
	},
	"S6": {
		inside: func(f *signalFacts) {
			f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
				on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
		},
		past: func(f *signalFacts) {
			f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
				on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
		},
	},
	"S7": {
		inside: func(f *signalFacts) {
			f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
		},
		past: func(f *signalFacts) {
			f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
		},
	},
	"S8": {
		inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
		past:   func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
	},
	"S9": {
		inside: func(f *signalFacts) {
			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
				First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
		},
		past: func(f *signalFacts) {
			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
				First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
		},
	},
	"S10": {
		inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
		past:   func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
	},
	"S11": {
		inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
		past:   func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
	},
	"S12": {
		inside: func(f *signalFacts) { f.lease.renewed = f.now.Add(-15 * time.Second) },
		past:   func(f *signalFacts) { f.lease.renewed = f.now.Add(-16 * time.Second) },
	},
	"S13": {
		inside: func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 5) },
		past:   func(f *signalFacts) { f.staleRefusals = refusedBy(f.now, 41, 6) },
	},
	// Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not.
	"S15": {
		inside: func(f *signalFacts) {
			f.handActs = []link.HandAct{actByHand(f.now.Add(-15*24*time.Hour), "consumer-behind"),
				actByHand(f.now.Add(-time.Hour), "consumer-behind")}
		},
		past: func(f *signalFacts) {
			f.handActs = []link.HandAct{actByHand(f.now.Add(-13*24*time.Hour), "consumer-behind"),
				actByHand(f.now.Add(-time.Hour), "consumer-behind")}
		},
	},
}

// actByHand is one entry in the hand-act log, with its cause.
func actByHand(at time.Time, cause string) link.HandAct {
	return link.HandAct{ID: "act-" + at.Format("150405"), At: at, By: "jochen at a shell on the laptop",
		Verb: "broker consumer-reset", Args: []string{"EVENTS", "controller"}, Why: "it replayed a week", Cause: cause}
}

// **S15 names the cause and, where a healer answers it, that the healer was not enough.**
func TestARepeatedHandActNamesItsCauseAndItsHealer(t *testing.T) {
	now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
	f := calm(now)
	f.handActs = []link.HandAct{actByHand(now.Add(-2*time.Hour), "consumer-behind"),
		actByHand(now.Add(-time.Hour), "consumer-behind"), actByHand(now.Add(-time.Hour), "restarted the proxy"),
		actByHand(now.Add(-time.Minute), "restarted the proxy")}
	got := watchHandActs(f)
	if len(got) != 2 {
		t.Fatalf("%+v", got)
	}
	if got[0].Key() != "mesh.hand-acts.consumer-behind.healer-wanted" || !strings.Contains(got[0].Summary, "healer H4") {
		t.Errorf("a cause a healer answers: %s — %s", got[0].Key(), got[0].Summary)
	}
	if got[1].Key() != "mesh.hand-acts.restarted_the_proxy.healer-wanted" ||
		!strings.Contains(got[1].Summary, "a healer is wanted") {
		t.Errorf("a cause no healer answers: %s — %s", got[1].Key(), got[1].Summary)
	}
}

// standingSaid is a provider's failing word last said at a moment.
func standingSaid(at time.Time) conditions.Condition {
	return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
}

func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
	now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
	seen := map[string]bool{}
	for _, row := range signalsTable {
		t.Run(row.Row, func(t *testing.T) {
			if seen[row.Row] {
				t.Fatalf("%s is in the table twice", row.Row)
			}
			seen[row.Row] = true
			if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
				(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
				t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
			}
			if row.Deferred != "" {
				if row.watch != nil || row.Phase <= 1 {
					t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
				}
				if _, has := suppressions[row.Row]; has {
					t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
				}
				return
			}
			if row.watch == nil || row.needs == nil || row.newest == nil {
				t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
			}
			s, ok := suppressions[row.Row]
			if !ok {
				t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
			}
			if got := row.watch(calm(now)); len(got) != 0 {
				t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
			}
			inside := calm(now)
			s.inside(inside)
			if got := row.watch(inside); len(got) != 0 {
				t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
			}
			past := calm(now)
			s.past(past)
			got := row.watch(past)
			if len(got) == 0 {
				t.Fatalf("%s raised nothing past its bound", row.Row)
			}
			for _, o := range got {
				if !slices.Contains(kindsOf(row), o.Kind) {
					t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
				}
				if o.Severity != row.Severity {
					t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
				}
				if strings.TrimSpace(o.Summary) == "" {
					t.Errorf("%s raised a condition that says nothing", row.Row)
				}
			}

			// Through the store: raised past the bound, cleared when the signal returns.
			store := conditions.NewInMemory()
			told := &conditions.Told{}
			k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
				Now: func() time.Time { return now }})
			defer k.Close(context.Background())
			w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
			w.see(t.Context(), past)
			open, err := k.Open(t.Context())
			if err != nil || len(open) != len(got) {
				t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
			}
			w.see(t.Context(), calm(now))
			if open, _ := k.Open(t.Context()); len(open) != 0 {
				t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
			}
		})
	}
	for name := range suppressions {
		if !seen[name] {
			t.Errorf("a suppression for %s, which the table does not have", name)
		}
	}
}

// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
	now := time.Now()
	f := calm(now)
	f.machines[0].lastHeard = now.Add(-31 * time.Minute)
	f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
	got := watchHeartbeats(f)
	if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
		t.Fatalf("%+v", got)
	}
}

// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
	now := time.Now()
	f := calm(now)
	f.machines[0].lastHeard = now.Add(-2 * time.Hour)
	f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
	f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
	if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
		t.Fatalf("a sleeping machine raised %+v", got)
	}
	f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
	if got := watchHeartbeats(f); len(got) != 1 {
		t.Fatalf("a woken machine silent past its bound raised %+v", got)
	}
}

// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
	now := time.Now()
	store := conditions.NewInMemory()
	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
	defer k.Close(context.Background())
	w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
	silent := calm(now)
	silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
	w.see(t.Context(), silent)
	blind := calm(now)
	blind.machines, blind.machinesErr = nil, errors.New("the store is away")
	w.see(t.Context(), blind)
	open, err := k.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	var keys []string
	for _, c := range open {
		keys = append(keys, c.Key)
	}
	for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
		if !slices.Contains(keys, want) {
			t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
		}
	}
	w.see(t.Context(), calm(now))
	if open, _ := k.Open(t.Context()); len(open) != 0 {
		t.Fatalf("seeing again left open %+v", open)
	}
}

// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
// every machine silent.
func TestAControllerStandingBySaysNothing(t *testing.T) {
	store := conditions.NewInMemory()
	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
	defer k.Close(context.Background())
	w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
	w.tick(t.Context())
	if w.lastTick().IsZero() {
		t.Fatal("a tick standing by was not counted")
	}
	if open, _ := k.Open(t.Context()); len(open) != 0 {
		t.Fatalf("%+v", open)
	}
}

// **A stale writer is named** (novox/hq to-be 45 §3, S13): by the controller instance that held the epoch
// its refused declarations claimed, and how that epoch ended — the question issue 204 could not answer.
func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
	now := time.Now()
	f := calm(now)
	ended := now.Add(-time.Minute)
	f.staleRefusals = refusedBy(now, 41, 6)
	f.epochs[41] = inventory.Epoch{Epoch: 41, Instance: "controller@anchor pid 7 since 2026-10-06T10:00:00Z",
		Host: "anchor", Ended: &ended, How: inventory.EpochExpired}
	got := watchStaleRefusals(f)
	if len(got) != 1 || got[0].Key() != "core.controller.epoch-41.stale-writer" ||
		!strings.Contains(got[0].Summary, "pid 7") || !strings.Contains(got[0].Summary, "expired") ||
		got[0].Machine != "anchor" || !slices.Equal(got[0].Also, []string{"laptop"}) {
		t.Fatalf("the stale writer is not named: %+v", got)
	}
	// An account the controller refused names the machine whose node-engine sent it.
	f.staleRefusals = []link.WriterRefusals{{Writer: link.WriterNodeEngine("laptop"), Count: 6,
		Receivers: []string{"controller"}, Last: now}}
	got = watchStaleRefusals(f)
	if len(got) != 1 || got[0].Key() != "machine.laptop.stale-writer" || got[0].Machine != "laptop" {
		t.Fatalf("a node-engine sending older accounts is not named: %+v", got)
	}
}

// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
	now := time.Now()
	f := calm(now)
	expired := now.Add(-59 * time.Minute)
	f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
		Ended: &expired, How: inventory.EpochExpired}}
	got := watchLease(f)
	if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
		got[0].Severity != conditions.Urgent {
		t.Fatalf("a holder that stopped renewing was not said: %+v", got)
	}
	long := now.Add(-61 * time.Minute)
	f.lease.ended[0].Ended = &long
	if got := watchLease(f); len(got) != 0 {
		t.Fatalf("a loss an hour old is still said: %+v", got)
	}
	f.lease.ended[0].How, f.lease.ended[0].Ended = inventory.EpochReleased, &expired
	if got := watchLease(f); len(got) != 0 {
		t.Fatalf("a lease given back is said as lost: %+v", got)
	}
	f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
	if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
		t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
	}
	f.lease = leaseFacts{unleased: "the bus refused the key"}
	if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.unleased" {
		t.Fatalf("serving without the lease was not said: %+v", got)
	}
}