mesh-controller/cmd/mesh-controller/signals_test.go

package main

import (
	"context"
	"errors"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/link"
)

// The test generated from the signals table (novox/hq to-be 45 §3, ADR 0227 rule 5, "how it is
// checked"): **every row is walked**. A watched row's signal is suppressed just inside its bound —
// nothing raised — and just past it — its condition raised, with its kind and severity — and restored,
// and the condition clears. A row that is not watched says why. A row added to the table without a
// suppression here fails, so the table cannot grow a watchdog nobody has seen fire.

// calm is a mesh whose every signal is fresh: one control node heard ten seconds ago, its last send
// reported, a plan a minute into its tier, the loop taking, the merges read, nothing asked, no call
// running, the self-check a minute old.
func calm(now time.Time) *signalFacts {
	return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
		machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
			every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
			lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
		plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
			entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
		loop:          loopFacts{took: now.Add(-time.Second), pending: 1},
		mergesPassed:  now.Add(-time.Minute),
		selfCheck:     selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute},
		lostConsumers: map[string]bool{}, staleRefusals: map[string]int{},
	}
}

// suppression is one row's signal held back: inside its bound, and past it.
type suppression struct{ inside, past func(f *signalFacts) }

// suppressions are every watched row's, by row.
var suppressions = map[string]suppression{
	"S1": {
		inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) },
		past:   func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) },
	},
	"S2": {
		inside: func(f *signalFacts) {
			f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false
		},
		past: func(f *signalFacts) {
			f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false
		},
	},
	"S3": {
		inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) },
		past:   func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) },
	},
	"S4": {
		inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) },
		past:   func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) },
	},
	"S5": {
		inside: func(f *signalFacts) {},
		past: func(f *signalFacts) {
			f.merges = []missedMerge{{Owner: "novox", Repo: "app", Base: "main", Commit: "c0ffee0011", At: f.now.Add(-11 * time.Minute)}}
		},
	},
	"S6": {
		inside: func(f *signalFacts) {
			f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
				on: "anchor", since: f.now.Add(-59 * time.Minute), bound: time.Hour}}
		},
		past: func(f *signalFacts) {
			f.asks = []askFacts{{id: "build-1", seat: "node-build-agent", what: "novox/app", state: link.AskInFlight,
				on: "anchor", since: f.now.Add(-61 * time.Minute), bound: time.Hour}}
		},
	},
	"S7": {
		inside: func(f *signalFacts) {
			f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-29 * time.Minute)}}
		},
		past: func(f *signalFacts) {
			f.calls = []link.Call{{ID: "call-1", Seat: "mesh-controller", Verb: "push", Started: f.now.Add(-31 * time.Minute)}}
		},
	},
	"S8": {
		inside: func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-29 * time.Minute))} },
		past:   func(f *signalFacts) { f.standings = []conditions.Condition{standingSaid(f.now.Add(-31 * time.Minute))} },
	},
	"S9": {
		inside: func(f *signalFacts) {
			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
				First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-61 * time.Minute), Count: 1}}
		},
		past: func(f *signalFacts) {
			f.advisories = []link.Advisory{{Kind: link.AdvisoryMaxDeliveries, ID: "EVENTS.anchor_shop", Said: "gave up",
				First: f.now.Add(-2 * time.Hour), Last: f.now.Add(-59 * time.Minute), Count: 1}}
		},
	},
	"S10": {
		inside: func(f *signalFacts) { f.selfCheck.last = f.now.Add(-9 * time.Minute) },
		past:   func(f *signalFacts) { f.selfCheck.last = f.now.Add(-11 * time.Minute) },
	},
	"S11": {
		inside: func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-179 * time.Second) },
		past:   func(f *signalFacts) { f.machines[0].toolsHeard = f.now.Add(-181 * time.Second) },
	},
	"S13": {
		inside: func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 5} },
		past:   func(f *signalFacts) { f.staleRefusals = map[string]int{"anchor": 6} },
	},
}

// standingSaid is a provider's failing word last said at a moment.
func standingSaid(at time.Time) conditions.Condition {
	return conditions.Condition{Key: "provider.idp.anchor.app.failing", Kind: kindProviderFailing, LastObserved: at}
}

func TestEveryRowOfTheSignalsTableIsWatchedRaisedAndCleared(t *testing.T) {
	now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)
	seen := map[string]bool{}
	for _, row := range signalsTable {
		t.Run(row.Row, func(t *testing.T) {
			if seen[row.Row] {
				t.Fatalf("%s is in the table twice", row.Row)
			}
			seen[row.Row] = true
			if row.Signal == "" || row.Emitter == "" || row.Trigger == "" || row.Bound == "" || row.Kind == "" ||
				(row.Severity != conditions.Urgent && row.Severity != conditions.Warning) {
				t.Fatalf("%s does not say what it expects, from whom, within what, and what it raises: %+v", row.Row, row)
			}
			if row.Deferred != "" {
				if row.watch != nil || row.Phase <= 1 {
					t.Fatalf("%s is deferred and watched, or deferred out of Phase 1's own rows: %+v", row.Row, row)
				}
				if _, has := suppressions[row.Row]; has {
					t.Fatalf("%s is deferred and has a suppression: one of the two is stale", row.Row)
				}
				return
			}
			if row.watch == nil || row.needs == nil || row.newest == nil {
				t.Fatalf("%s is watched and lacks its watch, its needs or its newest", row.Row)
			}
			s, ok := suppressions[row.Row]
			if !ok {
				t.Fatalf("%s has no suppression in this test: a watchdog nobody has seen fire", row.Row)
			}
			if got := row.watch(calm(now)); len(got) != 0 {
				t.Fatalf("%s raised on a calm mesh: %+v", row.Row, got)
			}
			inside := calm(now)
			s.inside(inside)
			if got := row.watch(inside); len(got) != 0 {
				t.Fatalf("%s raised inside its bound: %+v", row.Row, got)
			}
			past := calm(now)
			s.past(past)
			got := row.watch(past)
			if len(got) == 0 {
				t.Fatalf("%s raised nothing past its bound", row.Row)
			}
			for _, o := range got {
				if !slices.Contains(kindsOf(row), o.Kind) {
					t.Errorf("%s raised %q, which is not its kind %q", row.Row, o.Kind, row.Kind)
				}
				if o.Severity != row.Severity {
					t.Errorf("%s raised %s, the table says %s", row.Row, o.Severity, row.Severity)
				}
				if strings.TrimSpace(o.Summary) == "" {
					t.Errorf("%s raised a condition that says nothing", row.Row)
				}
			}

			// Through the store: raised past the bound, cleared when the signal returns.
			store := conditions.NewInMemory()
			told := &conditions.Told{}
			k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told,
				Now: func() time.Time { return now }})
			defer k.Close(context.Background())
			w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
			w.see(t.Context(), past)
			open, err := k.Open(t.Context())
			if err != nil || len(open) != len(got) {
				t.Fatalf("%s past its bound left %d open (%v), want %d", row.Row, len(open), err, len(got))
			}
			w.see(t.Context(), calm(now))
			if open, _ := k.Open(t.Context()); len(open) != 0 {
				t.Fatalf("%s's condition stayed open after the signal returned: %+v", row.Row, open)
			}
		})
	}
	for name := range suppressions {
		if !seen[name] {
			t.Errorf("a suppression for %s, which the table does not have", name)
		}
	}
}

// **The control node silent for half an hour is urgent** (S1); any other machine stays a warning.
func TestTheControlNodeSilentIsUrgentAfterHalfAnHour(t *testing.T) {
	now := time.Now()
	f := calm(now)
	f.machines[0].lastHeard = now.Add(-31 * time.Minute)
	f.machines = append(f.machines, machineFacts{name: "laptop", lastHeard: now.Add(-31 * time.Minute)})
	got := watchHeartbeats(f)
	if len(got) != 2 || got[0].Severity != conditions.Urgent || got[1].Severity != conditions.Warning {
		t.Fatalf("%+v", got)
	}
}

// **A machine that said it sleeps is not silent** (ADR 0211), nor late to report; one that woke is.
func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
	now := time.Now()
	f := calm(now)
	f.machines[0].lastHeard = now.Add(-2 * time.Hour)
	f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
	f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
	if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
		t.Fatalf("a sleeping machine raised %+v", got)
	}
	f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
	if got := watchHeartbeats(f); len(got) != 1 {
		t.Fatalf("a woken machine silent past its bound raised %+v", got)
	}
}

// **A watchdog that cannot see says so, and clears nothing it raised** (ADR 0227 rule 4): the store
// unreadable is a probe-failed of its own, and the machine's silence stays open until it can see again.
func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) {
	now := time.Now()
	store := conditions.NewInMemory()
	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
	defer k.Close(context.Background())
	w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
	silent := calm(now)
	silent.machines[0].lastHeard = now.Add(-10 * time.Minute)
	w.see(t.Context(), silent)
	blind := calm(now)
	blind.machines, blind.machinesErr = nil, errors.New("the store is away")
	w.see(t.Context(), blind)
	open, err := k.Open(t.Context())
	if err != nil {
		t.Fatal(err)
	}
	var keys []string
	for _, c := range open {
		keys = append(keys, c.Key)
	}
	for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} {
		if !slices.Contains(keys, want) {
			t.Errorf("%s is not open while the machines cannot be read: %v", want, keys)
		}
	}
	w.see(t.Context(), calm(now))
	if open, _ := k.Open(t.Context()); len(open) != 0 {
		t.Fatalf("seeing again left open %+v", open)
	}
}

// **A controller standing by sees nothing and says nothing**: it hears no heartbeat, and would call
// every machine silent.
func TestAControllerStandingBySaysNothing(t *testing.T) {
	store := conditions.NewInMemory()
	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store})
	defer k.Close(context.Background())
	w := &watchdogs{keeper: k, started: time.Now(), acting: func() bool { return false }}
	w.tick(t.Context())
	if w.lastTick().IsZero() {
		t.Fatal("a tick standing by was not counted")
	}
	if open, _ := k.Open(t.Context()); len(open) != 0 {
		t.Fatalf("%+v", open)
	}
}