Files
mesh-delivery/modules/mesh-delivery-watchman/internal/watch/findings_test.go
T
jschoubben d7b53bf2f1 D1 (watchman): the watchman names its own module from its credential's
Its finding observer, its dead-letter consumer, its reader's module and its tools' first views take the credential's module (the tools: the module the runtime serves them as) instead of constants of its name; a guard test finds none in its code.

Builds: design 47 G A-296, §428

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 22:46:17 +01:00

614 lines
36 KiB
Go

package watch
import (
"encoding/json"
"errors"
"strings"
"sync/atomic"
"testing"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
)
func mustEncode(t *testing.T, f Finding) []byte {
t.Helper()
v, err := EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
return v
}
// expire hears the expiry of the pulse at pk.
func expire(w *Watchman, c *clock, pk string, after time.Duration) {
c.t = c.t.Add(after)
w.Pulse(pk, Expired, c.t)
}
// The watchman's findings (novox/hq ADR 0564 §2, §4, §5, issue 1952 WM2-WM5): per row of the signals table a key not renewed
// writes its finding and a renewed one none; the haves' rows; stuck keys and presence as its own findings; the dead letters.
func TestTheWatchmansFindings(t *testing.T) {
cases := []struct {
name string
run func(t *testing.T, w *Watchman, o *fakeOut, c *clock)
// want are the finding keys standing at the end, with their severity.
want map[string]string
}{
{"ADR 0633 §5 (1585/T21): a revision waiting for an indexer build is a finding past 60 minutes from its since, not within; a group past 30", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp() // a live message, heard after the replay
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
t.Fatalf("a revision's wait: ttls %v, findings %v", o.ttls, o.findings)
}
w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0))
if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound {
t.Fatalf("a group's wait: ttls %v", o.ttls)
}
expire(w, c, signalKey("take-in", groupKey), TakeInGroupBound)
if len(o.findings) != 1 {
t.Fatalf("findings at 30 minutes: %v", o.findings)
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-TakeInGroupBound)
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning, FindingKey(ScopeDelivery, groupKey, KindTakeInWaits): Warning}},
{"ADR 0633 §5 (1585/T21): the take-in.taken of a named indexer, a hand take-in or a drop clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp() // a live message, heard after the replay
for _, by := range []string{"", "operator@shanks"} {
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
if len(o.findings) != 1 {
t.Fatalf("findings past the bound: %v", o.findings)
}
taken, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInTaken, Key: revKey, What: takein.TakeInRevision, Since: t0,
Merges: []takein.Merge{{Repository: "novox/a", Merge: "c0ffee"}}, By: by, Why: map[bool]string{true: "dropped", false: ""}[by != ""], Dropped: by != ""})
if err != nil {
t.Fatal(err)
}
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), taken)
c.t = t0
}
}, map[string]string{}},
{"ADR 0633 §5 (1585/T21): a group's take-in.taken ends its 30 minutes, and the revision then pending under m.<seq> starts its own 60", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp() // a live message, heard after the replay
w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0))
expire(w, c, signalKey("take-in", groupKey), 25*time.Minute)
w.TakeIn(takeInEvent(t, takein.TakeInTaken, groupKey), takeInBody(t, takein.TakeInTaken, groupKey, t0))
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, c.t))
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound {
t.Fatalf("after the hand-over: findings %v, ttls %v", o.findings, o.ttls)
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said once the replay has caught up, not before", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour)))
if len(o.findings) != 0 || len(o.pulses) != 0 {
t.Fatalf("a replayed old wait before the replay caught up: findings %v, pulses %v", o.findings, o.pulses)
}
w.TakeInCaughtUp()
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"1966/AC195-5: a seed ends the finding and the held wait of a key indexer_pending no longer names, and keeps one it names with its own since", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
// A previous run's finding for revKey stands in the bucket (read at the start); a wait of staleKey is held from an event.
old := Finding{Scope: ScopeDelivery, ID: revKey, Kind: KindTakeInWaits, Severity: Warning, Source: "take-in", Since: t0.Add(-3 * time.Hour), Observer: testModule, Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"}
value, err := EncodeFinding(old)
if err != nil {
t.Fatal(err)
}
o.findings[old.Key()] = old
w.SeedFinding(old.Key(), value)
const staleKey = "m.00000000000000000099"
w.TakeInCaughtUp()
w.TakeIn(takeInEvent(t, takein.TakeInPending, staleKey), takeInBody(t, takein.TakeInPending, staleKey, c.t))
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: groupKey, What: takein.TakeInGroup, Keys: []string{groupKey},
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Since: c.t.Add(-10 * time.Minute)}}})
if _, ok := o.findings[old.Key()]; ok || w.signals[signalKey("take-in", staleKey)].Finding.ID != "" || w.signals[signalKey("take-in", revKey)].Finding.ID != "" {
t.Fatalf("a wait the answer does not name stands: findings %v, signals %v", o.findings, w.signals)
}
if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound-10*time.Minute {
t.Fatalf("the named wait: ttl %v, want the rest of its bound from its own since", o.ttls[signalKey("take-in", groupKey)])
}
}, map[string]string{}},
{"1966/AC195-4: a taken event for a key the watchman holds no wait for changes no finding and no signal", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp()
w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0))
ttls, pulses := len(o.ttls), len(o.pulses)
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0))
if o.findingWrites != 0 || len(o.ttls) != ttls || len(o.pulses) != pulses || o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound {
t.Fatalf("a taken for an unheld key: findings written %d, ttls %v, pulses %v", o.findingWrites, o.ttls, o.pulses)
}
}, map[string]string{}},
{"ADR 0633 §5 (1585/T21): a replayed pending followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour)))
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0.Add(-2*time.Hour)))
w.TakeInCaughtUp()
if o.findingWrites != 0 || len(o.pulses) != 0 {
t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses)
}
}, map[string]string{}},
{"ADR 0633 §5 (1585/T21): a wait the indexer holds at a restart within its bound is seeded from indexer_pending and raised past it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
// The restarted watchman's durable consumer hears nothing of the pending event it acknowledged before; the wait
// began ten minutes ago, and the indexer still holds it.
since := c.t.Add(-10 * time.Minute)
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey},
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, WaitsFor: "src2:x", Since: since}}})
w.TakeInCaughtUp()
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute {
t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)])
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute)
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp()
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t,
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"})
if err != nil {
t.Fatal(err)
}
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body)
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings)
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
for _, f := range o.findings {
if !strings.Contains(f.Summary, "cannot learn the build it runs") {
t.Errorf("finding %q does not say why", f.Summary)
}
}
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp() // a live message, heard after the replay
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
if len(o.findings) != 1 {
t.Fatalf("findings past the bound: %v", o.findings)
}
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0))
if len(o.findings) != 0 {
t.Fatalf("the drop left %v", o.findings)
}
next := "m.00000000000000000043"
w.TakeIn(takeInEvent(t, takein.TakeInPending, next), takeInBody(t, takein.TakeInPending, next, t0))
}, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}},
{"design 47 G §500: manifest-refused raises the refusal, manifest-accepted clears it and carries an outgrown manifest's problems", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "nats", Problems: []string{"removes a stream"}})
w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "power", Problems: []string{"shows nothing"}})
k := FindingKey(ScopeModule, "nats", KindManifestRefused)
if f := o.findings[k]; !strings.Contains(f.Summary, "refused the manifest of nats: removes a stream") {
t.Fatalf("findings = %v", o.findings)
}
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "nats", Digest: strings.Repeat("d", 64)})
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power", Digest: strings.Repeat("d", 64)})
// ADR 0579 §1: an accepted manifest a stricter judge outgrew is a warning until mended.
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("d", 64), Problems: []string{"says no credential form"}})
if f := o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)]; f.Severity != Warning || !strings.Contains(f.Summary, "says no credential form") {
t.Fatalf("findings = %v", o.findings)
}
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("e", 64), Problems: []string{}})
// The start seeds from the indexer's verdicts: the newest of each module counts.
w.SeedVerdicts([]VerdictRow{
{Module: "nats", State: meshcatalogclient.VerdictRefused, Problems: []string{"old"}, At: t0},
{Module: "nats", State: meshcatalogclient.VerdictAccepted, At: t0.Add(time.Hour)},
{Module: "power", State: meshcatalogclient.VerdictRefused, Problems: []string{"shows nothing"}, At: t0},
{Module: "node-tools", State: meshcatalogclient.VerdictAccepted, Warnings: []string{"w"}, At: t0}})
if len(o.findings) != 2 || o.findings[FindingKey(ScopeModule, "power", KindManifestRefused)].Kind == "" || o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)].Kind == "" {
t.Fatalf("findings = %v", o.findings)
}
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power"})
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools"})
}, map[string]string{}},
{"DLV-W1, 1966/AC37-AC38 (ADR 0595 §4, design 25 § 12.2): an accepted entry the bus module says nothing of in five minutes is overdue until created; a refused one stands until created; a later verdict no longer asking ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
accepted := func(locals ...string) meshcatalog.ManifestAcceptedEvent {
a := meshcatalog.ManifestAcceptedEvent{Module: "mesh-knowledge", Digest: strings.Repeat("d", 64)}
for _, l := range locals {
a.BusState = append(a.BusState, meshcatalog.ManifestAcceptedEventBusStateItem{Name: l, Storage: "ledger"})
}
return a
}
w.Accepted(accepted("records", "drafts", "kept"))
pk := func(local string) string { return signalKey(rowBusState, busStateID("mesh-knowledge", local)) }
if o.ttls[pk("records")] != OverdueBound {
t.Fatalf("no five-minute pulse for records: %v", o.ttls)
}
w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "drafts"})
expire(w, c, pk("drafts"), 5*time.Minute) // created in time: no finding
expire(w, c, pk("records"), 0)
overdue := o.findings[overdueKey("mesh-knowledge", "records")]
if overdue.Kind != KindBusStateOverdue || !strings.Contains(overdue.Summary, "mesh-knowledge's records") {
t.Fatalf("findings = %v", o.findings)
}
w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "records"})
if _, open := o.findings[overdueKey("mesh-knowledge", "records")]; open {
t.Fatal("created did not clear overdue")
}
// Review round 1: created is said once per entry, so a second accepted manifest with the same entries waits on
// none of them, and the mark that says so has no time to live.
delete(o.pulses, pk("records")) // expired above; the fake keeps an expired pulse
w.Accepted(accepted("records", "drafts"))
for _, local := range []string{"records", "drafts"} {
if _, waits := o.pulses[pk(local)]; waits {
t.Fatalf("a second accepted manifest waits again on %s, said created already", local)
}
if mk := signalKey(rowBusCreated, busStateID("mesh-knowledge", local)); o.ttls[mk] != 0 || o.pulses[mk].Owes == "" {
t.Fatalf("the mark of %s created is %v with time to live %v", local, o.pulses[mk], o.ttls[mk])
}
}
w.BrokerRefused(BrokerSaid{Module: "mesh-knowledge", Local: "kept", Why: "the cap is above the mesh's bound"})
if f := o.findings[refusedKey("mesh-knowledge", "kept")]; f.Severity != Urgent || !strings.Contains(f.Summary, "cap is above") {
t.Fatalf("findings = %v", o.findings)
}
// A restart keeps both kinds: what ends them is heard.
if !signalKind(mustEncode(t, o.findings[refusedKey("mesh-knowledge", "kept")])) {
t.Error("a refused creation would be dropped at a restart")
}
// A later verdict asking for kept alone again waits on it again; one no longer asking for it ends it.
w.Accepted(accepted("kept", "fresh"))
w.Accepted(accepted("fresh"))
if _, open := o.findings[refusedKey("mesh-knowledge", "kept")]; open {
t.Fatal("a verdict no longer asking for kept left its finding")
}
if _, waits := o.pulses[pk("kept")]; waits {
t.Fatal("a verdict no longer asking for kept left its wait")
}
w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "fresh"})
// A refused create stands until created.
w.Accepted(accepted("gone"))
w.BrokerRefused(BrokerSaid{Module: "mesh-knowledge", Local: "gone"})
}, map[string]string{refusedKey("mesh-knowledge", "gone"): Urgent}},
{"ADR 0615 §5 (review round 1): a consumer the watchman cannot bind is family-not-heard naming its stream, until it binds", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.NotHeard("EVENTS", "what it carries is heard once the grants part has made the consumer n1_mesh-delivery-watchman", errors.New("consumer not found"))
f := o.findings[heard.Key(testModule, "EVENTS")]
if f.Kind != heard.Kind || f.Severity != Warning || !strings.Contains(f.Summary, "does not hear EVENTS") || !strings.Contains(f.Summary, "consumer not found") {
t.Fatalf("findings = %v", o.findings)
}
w.NotHeard("membership", "a consumer not there yet is not opened", errors.New("no bucket"))
w.Heard("EVENTS")
w.Heard("membership")
// meshconditionsclient/heard Writer, as the indexer, the assigner and the builder use it: Bind writes the finding with why
// when the consumer does not bind and answers the error, deletes it when it binds, and with no findings
// state says it in the log alone.
at := time.Date(2026, 11, 4, 17, 0, 0, 0, time.UTC)
for _, row := range []struct {
name string
bindErr error
noState bool
want string
}{
{name: "not there yet", bindErr: errors.New("consumer not found"),
want: "mesh-delivery-indexer does not hear assigner-views: the assignments is the first view's alone (consumer not found)"},
{name: "bound"},
{name: "no findings state", bindErr: errors.New("consumer not found"), noState: true},
} {
key := heard.Key("mesh-delivery-indexer", "assigner-views")
kept := map[string][]byte{key: []byte("{}")}
var logs []string
hw := heard.Writer{Observer: "mesh-delivery-indexer", Now: func() time.Time { return at }, Logf: func(f string, a ...any) { logs = append(logs, f) }}
if !row.noState {
hw.Put = func(k string, v []byte) error { kept[k] = v; return nil }
hw.Delete = func(k string) error { delete(kept, k); return nil }
}
if err := hw.Bind("assigner-views", "the assignments is the first view's alone", func() error { return row.bindErr }); !errors.Is(err, row.bindErr) {
t.Errorf("%s: Bind answered %v, want %v", row.name, err, row.bindErr)
}
raw, stands := kept[key]
switch {
case row.noState && len(logs) == 0:
t.Errorf("%s: nothing said", row.name)
case !row.noState && row.want == "" && stands:
t.Errorf("%s: the finding stands once bound", row.name)
case row.want != "":
var f condseat.Finding
err := json.Unmarshal(raw, &f)
// Review round 2: it says it clears at the part's next start, since Bind runs at the start.
if err != nil || f.Kind != heard.Kind || f.Summary != row.want || f.Observer != "mesh-delivery-indexer" || !strings.Contains(f.Explanation, "next start") {
t.Errorf("%s: the finding %+v (%v), want summary %q", row.name, f, err, row.want)
}
}
}
}, map[string]string{}},
{"WM4: a stuck key not moved is its finding, and its move deletes it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
askProgress(w, "ask1", t0)
expire(w, c, pulseKeyOf(t, checksKey, "ask1"), 120*time.Second)
k := FindingKey(ScopeDelivery, stuckKeyOf(t, checksKey, "ask1"), KindStuck)
if f := o.findings[k]; f.Kind != KindStuck || !strings.Contains(f.Summary, "the ask ask1 to end on n1") {
t.Fatalf("findings = %v", o.findings)
}
askEnded(w, "ask1")
}, map[string]string{}},
{"WM2 S1: a node offline past its bound is silent, and back online it is not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Connect("node.ace")
w.Disconnect("node.ace")
expire(w, c, pulseKeyOf(t, presenceNS, "ace"), 120*time.Second)
if _, ok := o.findings["machine.ace.silent"]; !ok {
t.Fatalf("findings = %v", o.findings)
}
w.Connect("node.ace")
}, map[string]string{}},
{"WM2 S1: a node that said it sleeps is not silent", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Connect("node.lap")
w.Power("lap", "sleeping")
w.Disconnect("node.lap")
expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second)
}, map[string]string{}},
{"WM2 S1: a node back from sleep and still offline is silent after its bound", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Connect("node.lap")
w.Power("lap", "sleeping")
w.Disconnect("node.lap")
expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second)
w.Power("lap", "woke")
expire(w, c, pulseKeyOf(t, presenceNS, "lap"), 120*time.Second)
}, map[string]string{"machine.lap.silent": Warning}},
{"WM2 S11: a tool runner not heard for three intervals is silent; a heartbeat ends it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.ToolsAlive("ace", 0)
pk := signalKey("S11", "ace")
if o.ttls[pk] != 3*HeartbeatEvery {
t.Fatalf("ttl = %v", o.ttls[pk])
}
expire(w, c, pk, 3*HeartbeatEvery)
if _, ok := o.findings["machine.ace.tools-silent"]; !ok {
t.Fatalf("findings = %v", o.findings)
}
w.ToolsAlive("ace", 20*time.Second)
if o.ttls[pk] != time.Minute {
t.Fatalf("ttl = %v, want three of the 20 s it says", o.ttls[pk])
}
w.ToolsAlive("g14", 0) // renewed, never expired: no finding
}, map[string]string{}},
{"WM2 S10: the self-check not heard for three runs is urgent; not before its first run", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
if len(o.pulses) != 0 {
t.Fatal("a pulse before the first run")
}
w.Checked("novox")
expire(w, c, signalKey("S10", "doctor"), 3*SelfCheckEvery)
}, map[string]string{"core.doctor.self-check-silent": Urgent}},
{"WM3 S8: a failing word is its finding; not said again for 30 min it is silent; recovered ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
st, err := ReadStanding("mesh.mod.keycloak.event.provisioner.failing",
[]byte(`{"provider":"oidc","provider-node":"ace","consumer":"photos","node":"g14","class":"refused","attempts":4,"since":"2026-10-18T11:00:00Z"}`))
if err != nil {
t.Fatal(err)
}
w.Stood(st)
if f := o.findings["provider.keycloak.ace.photos.failing"]; f.Kind != KindProviderFailing || len(f.Also) != 1 {
t.Fatalf("findings = %v", o.findings)
}
expire(w, c, signalKey("S8", "keycloak.ace.photos"), ProviderSaysAgainWithin)
if _, ok := o.findings["provider.keycloak.ace.photos.silent"]; !ok {
t.Fatalf("findings = %v", o.findings)
}
st.Failing = false
w.Stood(st)
}, map[string]string{}},
{"WM3 S9: an advisory is a finding counted, ended by an hour's quiet", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
a, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.EVENTS.ace_web", []byte(`{"stream":"EVENTS","consumer":"ace_web"}`),
func(s, c string) string { return c + " on " + s })
if !ok {
t.Fatal("a mesh consumer's deletion is not read")
}
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.KV_x.abc", []byte(`{"stream":"KV_x","consumer":"abc"}`), nil); ok {
t.Fatal("a reader's own consumer is said")
}
// 1126/B9 (ADR 0590 clause 6): the ended controller's consumers are removed as hand acts, never lost; only its
// consumer on DEAD_LETTER_NOTICES is still read, which tells the watchman to take the notices.
for _, gone := range [][2]string{{"EVENTS", "controller"}, {"EVENTS", "controller-merges"}, {"CONTROL", "controller"}} {
body := []byte(`{"stream":"` + gone[0] + `","consumer":"` + gone[1] + `"}`)
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED."+gone[0]+"."+gone[1], body, func(s, c string) string { return c }); ok {
t.Errorf("the removal of %s.%s is said as a consumer lost", gone[0], gone[1])
}
}
if _, ok := ReadAdvisory("$JS.EVENT.ADVISORY.CONSUMER.DELETED.DEAD_LETTER_NOTICES.controller",
[]byte(`{"stream":"DEAD_LETTER_NOTICES","consumer":"controller"}`), func(s, c string) string { return c }); !ok {
t.Error("the removal of the controller's consumer on DEAD_LETTER_NOTICES is not read")
}
w.Advised(a)
w.Advised(a)
k := "bus.EVENTS.ace_web.consumer-lost"
if f := o.findings[k]; !strings.Contains(f.Summary, "2 times") || f.Machine != "ace" {
t.Fatalf("findings = %v", o.findings)
}
expire(w, c, signalKey("S9", k), AdvisoryQuiet)
}, map[string]string{}},
{"WM3 S21: a provisioner client's failed files are apply-failing past the bound; a silent machine's are not", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.Connect("node.ace")
w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: ProvFailed, Reason: "disk full", At: t0.Add(-10 * time.Minute)})
if f := o.findings["machine.ace.apply-failing"]; !strings.Contains(f.Summary, "provisioner has failed to place the files of web") {
t.Fatalf("findings = %v", o.findings)
}
w.Provisioned(ProvReport{Node: "g14", Module: "web", Status: ProvFailed, Reason: "x", At: t0.Add(-10 * time.Minute)})
w.Provisioned(ProvReport{Node: "ace", Module: "web", Status: "applied", At: t0})
}, map[string]string{}},
{"1954/D2: a provision in waits is its consumer's needs-operator; provided, it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
enc := func(state, why string) []byte {
b, err := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: state, Why: why, At: t0})
if err != nil {
t.Fatal(err)
}
return b
}
w.Provision("ace.photos.oidc", enc(meshprovisionerclient.Waits, "the operator gives the client's redirect"), Put)
w.Provision("ace.photos.db", enc(meshprovisionerclient.Provided, ""), Put)
if f := o.findings["module.photos.ace.needs-operator"]; !strings.Contains(f.Summary, "redirect") || f.Machine != "ace" {
t.Fatalf("findings = %v", o.findings)
}
w.Provision("ace.photos.oidc", enc(meshprovisionerclient.Provided, ""), Put)
w.Provision("ace.web.oidc", enc(meshprovisionerclient.Waits, "x"), Put)
w.Provision("ace.web.oidc", nil, Delete)
}, map[string]string{}},
{"WM3 S9: the bus module's closures, each counted once, ended an hour after the newest", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
cl := BusClosure{User: "ace.web", CID: 7, Reason: "Slow Consumer (Write Deadline)", Opened: t0.Add(-time.Hour), Closed: t0}
w.Closed(KindSlowConsumer, cl)
w.Closed(KindSlowConsumer, cl) // said again when the bus module starts
cl.CID = 8
w.Closed(KindSlowConsumer, cl)
f := o.findings["bus.ace.web.slow-consumer"]
if f.Machine != "ace" || !strings.Contains(f.Summary, "web on ace") || !strings.Contains(f.Summary, "2 times") {
t.Fatalf("findings = %v", o.findings)
}
expire(w, c, signalKey("S9", "bus.ace.web.slow-consumer"), AdvisoryQuiet)
}, map[string]string{}},
{"WM5: while letters are held DEAD_LETTERS is counted again at its own pulse's expiry, and at a letter's age", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
var n atomic.Int32
w.SetRecount(func() { n.Add(1) })
w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 1}, func(s, c string) string { return c })
if o.ttls[signalKey("dead", "recount")] != DeadRecountEvery || !strings.Contains(o.findings["bus.EVENTS.ace_web.max-deliveries"].Said, "every 5m0s") {
t.Fatalf("pulses %v findings %v", o.ttls, o.findings)
}
w.DeadAging("EVENTS.ace_web", c.t.Add(time.Minute))
expire(w, c, signalKey("dead", "EVENTS.ace_web"), time.Minute)
expire(w, c, signalKey("dead", "recount"), DeadRecountEvery)
// The recount runs in a goroutine of its own: waited for, within seconds no loaded machine reaches.
for deadline := time.Now().Add(10 * time.Second); n.Load() < 2 && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
if n.Load() != 2 {
t.Fatalf("counted again %d times", n.Load())
}
w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 1}, func(s, c string) string { return c }) // the count renews its pulse
w.DeadLettersHeld(map[string]int{}, nil)
if _, ok := o.pulses[signalKey("dead", "recount")]; ok {
t.Fatal("still counted again with nothing held")
}
}, map[string]string{}},
{"WM4: a finding read at the start and not observed again is deleted; a signal's stays", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
for _, f := range []Finding{
{Scope: ScopeDelivery, ID: "x.y", Kind: KindStuck, Severity: Warning, Summary: "old", Source: "stuck", Observer: testModule},
{Scope: ScopeMachine, ID: "ace", Kind: KindToolsSilent, Severity: Warning, Summary: "silent", Source: "S11", Observer: testModule},
} {
v, _ := EncodeFinding(f)
o.findings[f.Key()], _ = DecodeFinding(v)
w.SeedFinding(f.Key(), v)
}
w.Reconcile()
}, map[string]string{"machine.ace.tools-silent": Warning}},
{"WM5: a consumer DEAD_LETTERS holds for is its finding until it holds none", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
who := func(s, c string) string { return c }
w.DeadLettersHeld(map[string]int{"EVENTS.ace_web": 2, "SEAT_X.y_worker": 1}, who)
if f := o.findings["bus.EVENTS.ace_web.max-deliveries"]; f.Machine != "ace" || !strings.Contains(f.Summary, "2 messages") {
t.Fatalf("findings = %v", o.findings)
}
w.DeadLettersHeld(map[string]int{"SEAT_X.y_worker": 1}, who)
w.NoticesWaiting(true)
}, map[string]string{"bus.SEAT_X.y_worker.max-deliveries": Warning, "bus.DEAD_LETTER_NOTICES.controller.notices-waiting": Warning}},
{"ADR 0603 §8: a want held by a rule and a provision asked each raise a finding naming the wait past their bound, and clear when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.SetStepBound(30 * time.Minute)
held, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it yet"})
released, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Self: true})
asked, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, At: t0})
provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Provided, At: t0})
w.EngineRunning("ace", "web", []byte(`{"fingerprint":"`+fpOld+`"}`)) // the node runs what the held want names: the hold is still the wait
w.Want("want.ace.web", held, Put, t0)
w.Provision("ace.photos.oidc", asked, Put)
waits := map[string]string{AssignerWantsBucket: "want.ace.web", ProvisionsBucket: "ace.photos.oidc"}
if len(o.pulses) != 2 || len(o.findings) != 0 {
t.Fatalf("pulses %v findings %v", o.pulses, o.findings)
}
// The keys are written again as their parts move on, with new words and a new time: the wait still began at t0.
c.t = c.t.Add(20 * time.Minute)
held2, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace runs it, not healthy yet"})
asked2, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, Why: "the provider is applying", At: c.t})
w.Want("want.ace.web", held2, Put, c.t)
w.Provision("ace.photos.oidc", asked2, Put)
for bucket, key := range waits {
expire(w, c, pulseKeyOf(t, bucket, key), 31*time.Minute)
}
for bucket, key := range waits {
f := o.findings[FindingKey(ScopeDelivery, stuckKeyOf(t, bucket, key), KindStuck)]
if f.Kind != KindStuck || !f.Since.Equal(t0) || !strings.Contains(f.Summary, "bound 30m0s") {
t.Fatalf("%s: findings = %v", bucket, o.findings)
}
}
w.Want("want.ace.web", released, Put, c.t)
w.Provision("ace.photos.oidc", provided, Put)
}, map[string]string{}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
w, o, c := setup()
tc.run(t, w, o, c)
got := map[string]string{}
for k, f := range o.findings {
got[k] = f.Severity
if f.Key() != k || f.Observer != testModule {
t.Errorf("finding %s = %+v", k, f)
}
}
if len(got) != len(tc.want) {
t.Fatalf("findings = %v, want %v", got, tc.want)
}
for k, sev := range tc.want {
if got[k] != sev {
t.Fatalf("findings = %v, want %v", got, tc.want)
}
}
})
}
}
const (
revKey = "m.00000000000000000042"
groupKey = "g.feat/take-in"
)
func takeInEvent(t *testing.T, kind, key string) string {
t.Helper()
e, err := takein.TakeInEvent(kind)
if err != nil {
t.Fatal(err)
}
return e
}
// takeInBody is a take-in event's body for a key: a group's when it starts with g.
func takeInBody(t *testing.T, kind, key string, since time.Time) []byte {
t.Helper()
body := takein.TakeIn{Kind: kind, Key: key, What: takein.TakeInRevision, Since: since,
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}}
if key[0] == 'g' {
body.What, body.Group = takein.TakeInGroup, "feat/take-in"
} else if kind == takein.TakeInPending {
body.WaitsFor = "src2:abc"
}
b, err := takein.EncodeTakeIn(body)
if err != nil {
t.Fatal(err)
}
return b
}
// Design 47 G §689, A-14, A-22: a token other than the kind goes into the contract finding's id and reads back as it was
// written; the finding's time is its `since`; the resolved words are not on the wire.
func TestAFindingIsTheContractsWithItsRefInItsID(t *testing.T) {
since := time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC)
for _, f := range []Finding{
{Scope: ScopeModule, ID: "web.ace", Token: "unhealthy", Kind: KindStuck, Machine: "ace", Severity: Warning, Source: "stuck", Since: since, Summary: "s", Said: "said"},
{Scope: ScopeModule, ID: "web", Token: KindBusStateOverdue + "-state", Kind: KindBusStateOverdue, Severity: Warning, Source: "bus", Since: since, Summary: "s"},
{Scope: ScopeMachine, ID: "ace", Kind: KindSilent, Severity: Warning, Source: "S1", Since: since, Summary: "s", Needs: "mend"},
} {
c := f.Contract()
wantID := f.ID
if f.Token != "" && f.Token != f.Kind {
wantID = f.ID + "." + f.Token
}
if c.ID != wantID || c.Kind != f.Kind || !c.At.Equal(since) || !c.Since.Equal(since) || c.Subject.ID != wantID {
t.Errorf("%+v: contract %+v, want id %q, kind %q, at and since %v", f, c, wantID, f.Kind, since)
}
raw, err := EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
if back, err := DecodeFinding(raw); err != nil || back.ID != f.ID || back.Token != f.Token || back.Said != f.Said || back.Needs != f.Needs {
t.Errorf("%+v read back as %+v (%v)", f, back, err)
}
if strings.Contains(string(raw), "esolved") {
t.Errorf("%+v: resolved words are on the wire: %s", f, raw)
}
}
}