package main import ( "encoding/json" "errors" "strings" "testing" "time" ) // The tests of novox/hq ADR 0308 "How it is checked" §2–§6, with the controller's walk events given by a fake // source: the body the controller's plan-moved carries at a walk's start and end (ADR 0308 §1). // walkBody is a plan-moved body as the controller says a walk's start or end (mesh-controller pull request 247). func walkBody(t *testing.T, id, phase, outcome string, at time.Time, bound int, touches ...Touch) []byte { t.Helper() b := map[string]any{"id": id, "state": "walking", "walk": id, "phase": phase, "touches": touches, "updated": at.Format(time.RFC3339Nano), "budget_class": "leaf"} if phase == WalkStarted { b["started_at"] = at.Format(time.RFC3339Nano) } else { b["ended_at"] = at.Format(time.RFC3339Nano) } if outcome != "" { b["outcome"] = outcome } if bound > 0 { b["bound_seconds"] = bound } raw, err := json.Marshal(b) if err != nil { t.Fatal(err) } return raw } // walk says a walk's start or end to the router, through the decoder the bus's events go through. func walk(t *testing.T, h *Holder, id, phase, outcome string, at time.Time, bound int, touches ...Touch) { t.Helper() ev, ok, err := DecodeWalk(walkBody(t, id, phase, outcome, at, bound, touches...)) if err != nil || !ok { t.Fatalf("the walk event was not read: %v %v", ok, err) } h.Walked(ev) } func novox(modules ...string) Touch { return Touch{Machine: "novox", Modules: modules} } // heard says whether any message on the channels carries the words: sent, edited or said again in place. func heard(words string, chs ...*fakeChannel) bool { for _, ch := range chs { for _, m := range ch.sends { if strings.Contains(m.Text(), words) { return true } } for _, m := range ch.edits { if strings.Contains(m.Text(), words) { return true } } for _, r := range ch.replaced { if strings.Contains(r.m.Text(), words) { return true } } } return false } // outcomes are the history's outcomes for a key, oldest first. func outcomes(h *Holder, key string) []string { var out []string h.mu.Lock() defer h.mu.Unlock() for _, s := range h.recent { if s.Key == key { out = append(out, s.Outcome) } } return out } func has(list []string, want string) bool { for _, s := range list { if s == want { return true } } return false } // §2: during a walk, a message about a machine or a module it touches is held, not sent, and the channel's // history says it was held — a module's condition, a machine's, and a module's notice alike. func TestDuringAWalkWhatItTouchesIsHeldAndTheHistorySaysSo(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("mesh-tools", "node-tools")) mod := cond("module.node-tools.novox.unhealthy", Urgent, "the tool runner on novox does not answer") mod.Machine, mod.Raised = "novox", c.now() h.Condition(EventRaised, mod) machine := cond("machine.novox.claude-code.agent-home.data-shrank", Urgent, "the agent's home shrank") machine.Raised = c.now() h.Condition(EventRaised, machine) if _, err := h.Notify("tools.slow", Urgent, "tools answer slowly", "node-tools", false, "tools slow", "Needs you: look at the tools."); err != nil { t.Fatal(err) } runFor(h, c, 2*time.Minute) if saidOn(tg, dt) != 0 { t.Fatalf("held messages were said: telegram %+v, desk %+v", tg.sends, dt.sends) } for _, key := range []string{mod.Key, machine.Key, "notify.tools.slow"} { if !has(outcomes(h, key), "held") { t.Errorf("the history does not say %s was held: %v", key, outcomes(h, key)) } } if len(h.Open()) != 3 { t.Fatalf("held messages are not kept open: %+v", h.Open()) } } // §2: a message touched by two overlapping walks is held until the second of them ends; the first's end says // nothing of it. func TestAMessageTouchedByTwoOverlappingWalksIsHeldUntilTheSecondEnds(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) c.pass(time.Minute) walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"), Touch{Machine: "ace", Modules: []string{"node-tools"}}) k := cond("machine.novox.network", Urgent, "novox lost its default route") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-tools")) runFor(h, c, time.Minute) if heard(headlineOf(k.Key), tg, dt) { t.Fatalf("said at the first walk's end, while the second still holds it: %+v %+v", tg.sends, dt.sends) } if !heard("Delivered: mesh-tools on novox", tg, dt) { t.Fatalf("the first walk's end was not told: %+v", dt.sends) } walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, time.Minute) if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) { t.Fatalf("the second walk's end did not tell it: telegram %+v, desk %+v", tg.sends, dt.sends) } } // §3: an ask, a secret given, a message about a machine no open walk touches and a walk's own failure are // sent at once, while a walk touches the machine they are on. func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) { h, tg, dt, c, _, _ := newRouter(t) walk(t, h, "plan-9", WalkStarted, "", c.now(), 1200, novox("telegram", "mesh-tools")) for _, key := range []string{ "machine.novox.telegram.telegram-token.secret-given", // a secret given, on a touched machine "machine.novox.mounts.smb.secret-replaced", // a secret replaced "machine.ace.silent", // a machine no walk touches "delivery.leaf.over-budget", // the walk's own delivery "plan.plan-9.failed", // the walk itself } { k := cond(key, Urgent, "it happened") k.Raised = c.now() if !strings.HasPrefix(key, "machine.") { k.Machine = "novox" // a walk's own failure, on a machine it touches } h.Condition(EventRaised, k) runFor(h, c, time.Minute) stand(h, c) if !heard(headlineOf(key), tg, dt) { t.Errorf("%s was not sent at once during the walk: %v", key, outcomes(h, key)) } } // An ask's own message, and an ask about a held condition: both offered at once. submit(t, h, ConditionOwner, ackAsk(c, "own")) runFor(h, c, time.Minute) if !has(outcomes(h, "ask."+ConditionOwner+".own"), "sent") { t.Errorf("an ask waiting for the operator was held: %v", outcomes(h, "ask."+ConditionOwner+".own")) } held := cond("machine.novox.units", Urgent, "2 failed services on novox") held.Raised = c.now() h.Condition(EventRaised, held) runFor(h, c, time.Minute) if heard(headlineOf(held.Key), tg, dt) { t.Fatalf("a condition about a touched machine was not held") } about := ackAsk(c, "about") about.About = held.Key submit(t, h, ConditionOwner, about) runFor(h, c, time.Minute) if !heard(headlineOf(held.Key), dt) { t.Fatalf("the ask about a held condition was held: %v", outcomes(h, held.Key)) } } // §4: a walk's end is one message — its outcome, what it reached and how long it took — then each held message // still open, one line each; the one that cleared is only counted, and nothing more is said of them after. func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-3", WalkStarted, "", c.now(), 600, novox("mesh-tools")) still := cond("machine.novox.tools-silent", Warning, "the tools on novox are silent") still.Raised = c.now() h.Condition(EventRaised, still) gone := cond("module.mesh-tools.novox.unhealthy", Warning, "mesh-tools is unhealthy on novox") gone.Raised = c.now() h.Condition(EventRaised, gone) c.pass(time.Second) other := cond("machine.novox.core-behind", Urgent, "novox runs an older core") other.Raised = c.now() h.Condition(EventRaised, other) runFor(h, c, 2*time.Minute) h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()}) runFor(h, c, 5*time.Minute) if saidOn(tg, dt) != 0 { t.Fatalf("said during the walk: %+v %+v", tg.sends, dt.sends) } walk(t, h, "plan-3", WalkEnded, WalkFailed, c.now(), 0) runFor(h, c, time.Minute) // Urgent among what it holds: on every kind, one message each. if len(dt.sends) != 1 || len(tg.sends) != 1 { t.Fatalf("the end was not one message per kind: desk %+v, telegram %+v", dt.sends, tg.sends) } m := dt.sends[0] if m.Title != "Delivery failed: mesh-tools on novox" || !strings.Contains(m.Body, "It took 7 min.") { t.Fatalf("the end does not say its outcome, what it reached and how long: %q", m.Text()) } want := "Still open after it:\n- " + headlineOf(still.Key) + "\n- Urgent: " + headlineOf(other.Key) + "\n1 problem it raised resolved during it." if !strings.Contains(m.Body, want) || strings.Contains(m.Text(), headlineOf(gone.Key)) { t.Fatalf("the end message %q does not say %q", m.Body, want) } keyless(t, m) runFor(h, c, 20*time.Minute) if len(dt.sends) != 1 || len(tg.sends) != 1 || len(dt.replaced)+len(tg.replaced) != 0 { t.Fatalf("what the end told was said again: desk %+v, telegram %+v", dt.sends, tg.sends) } } // §4: a walk that delivered with nothing held and nothing open is one line, and nothing at all on a kind set // to say only what needs the operator. func TestADeliveredWalkWithNothingHeldIsOneLineOrNothing(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-4", WalkStarted, "", c.now(), 600, novox("mesh-tools"), Touch{Machine: "ace", Modules: []string{"mesh-tools"}}) runFor(h, c, 3*time.Minute) walk(t, h, "plan-4", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, time.Minute) if len(dt.sends) != 1 || len(tg.sends) != 0 { t.Fatalf("desk %+v, telegram %+v", dt.sends, tg.sends) } if m := dt.sends[0]; m.Title != "Delivered: mesh-tools on ace, novox, in 3 min" || m.Body != "" { t.Fatalf("not one line: %q", m.Text()) } h.NeedsOnly = func(kind string) bool { return kind == "desktop" } walk(t, h, "plan-5", WalkStarted, "", c.now(), 600, novox("mesh-tools")) runFor(h, c, 3*time.Minute) walk(t, h, "plan-5", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, time.Minute) if len(dt.sends) != 1 || len(tg.sends) != 0 { t.Fatalf("told on a kind that says only what needs the operator: desk %+v, telegram %+v", dt.sends, tg.sends) } if !has(outcomes(h, "walk.plan-5"), "quiet: this kind says only what needs the operator") { t.Fatalf("the history does not say why nothing was told: %v", outcomes(h, "walk.plan-5")) } } // §5: the bound is the controller's (bound_seconds: twice the largest budget of the walk's deliveries; none // given, 15 minutes). At it, what is held and still open is sent with the still-running line, and what // cleared is counted; before it, nothing; after it, the walk holds nothing more. func TestAHoldEndsAtTheWalksBound(t *testing.T) { for _, tc := range []struct { name string given int bound time.Duration }{ {"core: twice ten minutes", 1200, 20 * time.Minute}, {"leaf at five minutes", 600, 10 * time.Minute}, {"leaf at three minutes", 360, 6 * time.Minute}, {"no budget", 0, 15 * time.Minute}, } { t.Run(tc.name, func(t *testing.T) { h, tg, dt, c, _ := newHolder(t) started := c.now() walk(t, h, "plan-7", WalkStarted, "", started, tc.given, novox("node-tools")) open := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") open.Raised = c.now() h.Condition(EventRaised, open) gone := cond("module.node-tools.novox.unhealthy", Urgent, "node-tools is unhealthy") gone.Raised = c.now() h.Condition(EventRaised, gone) c.pass(30 * time.Second) h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()}) runFor(h, c, tc.bound-time.Minute-30*time.Second) if saidOn(tg, dt) != 0 { t.Fatalf("said before the bound: %+v %+v", tg.sends, dt.sends) } runFor(h, c, 2*time.Minute) if len(dt.sends) != 1 || len(tg.sends) != 1 { t.Fatalf("at the bound: desk %+v, telegram %+v", dt.sends, tg.sends) } m := dt.sends[0] since := started.In(h.zone()).Format("15:04") for _, want := range []string{"is still running, since " + since + ".", "- Urgent: " + headlineOf(open.Key), "1 problem it raised resolved during it."} { if !strings.Contains(m.Body, want) { t.Fatalf("the bound's message %q does not say %q", m.Text(), want) } } // Past its bound the walk holds nothing: a new message about what it touches goes at once. late := cond("machine.novox.disk", Urgent, "a disk fails on novox") late.Raised = c.now() h.Condition(EventRaised, late) runFor(h, c, time.Minute) if !heard(headlineOf(late.Key), dt) { t.Fatalf("held past the bound: %v", outcomes(h, late.Key)) } }) } } // §2, kept: what a walk holds survives the router's restart, and is told at the walk's end. func TestHeldMessagesSurviveARouterRestart(t *testing.T) { h, tg, dt, c, st := newHolder(t) walk(t, h, "plan-8", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf} if err := again.Load(); err != nil { t.Fatal(err) } again.Start() c.pass(2 * time.Minute) walk(t, again, "plan-8", WalkEnded, WalkDelivered, c.now(), 0) runFor(again, c, time.Minute) if !heard("- Urgent: "+headlineOf(k.Key), dt) || saidOn(tg, dt) != 2 { t.Fatalf("what was held before the restart was not told at the walk's end: desk %+v, telegram %+v", dt.sends, tg.sends) } } // §6, issue 531: the desk's notifiers on two machines start again a second apart, with four silenced messages // shown there — one of them withheld for its content, a file path — and a fifth open, asked there, and // withheld. Nothing is said again: no silenced, no withheld message; the pass runs once; and a router that // starts again reads it back from its state and makes it no more. func TestARestartSaysNothingItHasSaid(t *testing.T) { h, tg, dt, c, _, _ := newRouter(t) h.Start() c.pass(2 * time.Minute) keys := []string{"machine.novox.telegram.telegram-token.secret-given", "machine.novox.claude-code.agent-home.data-shrank", "machine.shanks.mounts.smb-password-recalbox.secret-given", "machine.ace.nzbgeek.api-key.secret-given"} var silenced []Condition for i, key := range keys { k := cond(key, Urgent, "it happened") if i == 1 { k.Explanation = "Needs you: look at /var/lib/agent-home, which shrank." } k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) stand(h, c) silenced = append(silenced, k) } withheldOpen := cond("machine.novox.units", Urgent, "2 failed services") withheldOpen.Explanation = "Needs you: read /etc/systemd/system/x.service." withheldOpen.Raised = c.now() h.Condition(EventRaised, withheldOpen) runFor(h, c, time.Minute) ask := ackAsk(c, "units") ask.About = withheldOpen.Key submit(t, h, ConditionOwner, ask) runFor(h, c, time.Minute) for _, k := range silenced { k.SilencedTill, k.SilencedWhy, k.At = c.now().Add(7*24*time.Hour), "known", c.now() h.Condition(EventChanged, k) } runFor(h, c, time.Minute) for _, key := range []string{silenced[1].Key, withheldOpen.Key} { if r := h.open[key]; r == nil || r.Refused == "" { t.Fatalf("the test's %s was not withheld for its content: %+v", key, r) } } sends, edits, replaced, tgSaid := len(dt.sends), len(dt.edits), len(dt.replaced), saidOn(tg) dt.edits = map[string]Message{} passes := func(h *Holder) int { return len(outcomes(h, "(answers lost)")) } // The notifiers on two machines started again a second apart: two standings, two times. c.pass(time.Minute) first := c.now() h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: first}) c.pass(time.Second) h.Flush() // the runtime's tick, between the two standings second := c.now() h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second}) runFor(h, c, time.Minute) if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid { t.Fatalf("the restart said again: sends %+v, edits %+v (was %d), replaced %+v", dt.sends[sends:], dt.edits, edits, dt.replaced[replaced:]) } if passes(h) != 1 { t.Fatalf("the pass ran %d times for one restart", passes(h)) } // The router starts again; the desk repeats its standing. again := &Holder{Away: tg, Desk: dt, Store: h.Store, Asks: h.Asks, Bench: h.Bench, Publish: h.Publish, Now: c.now, Logf: t.Logf, Zone: h.Zone, RootFree: h.RootFree, Node: h.Node} if err := again.Load(); err != nil { t.Fatal(err) } again.Start() again.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second}) runFor(again, c, 2*time.Minute) if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid || passes(again) != 1 { t.Fatalf("the router's restart made the pass again: %d passes, sends %+v, edits %+v", passes(again), dt.sends[sends:], dt.edits) } } // The controller's walk events are read as ADR 0308 §1 says them; any other plan-moved is not a walk's. func TestTheWalkEventsAreReadAsTheControllerSaysThem(t *testing.T) { at := time.Date(2026, 10, 12, 11, 12, 0, 0, time.UTC) ev, ok, err := DecodeWalk(walkBody(t, "plan-1", WalkStarted, "", at, 1200, novox("mesh-tools", "node-tools"))) if err != nil || !ok || ev.Walk != "plan-1" || ev.BoundSeconds != 1200 || ev.Class != "leaf" || !ev.At.Equal(at) || len(ev.Touches) != 1 || ev.Touches[0].Machine != "novox" || len(ev.Touches[0].Modules) != 2 { t.Fatalf("%+v %v %v", ev, ok, err) } // As mesh-controller's walk events (its pull request 247) say an end: ended_at, and carried beside modules. end, ok, err := DecodeWalk([]byte(`{"id":"plan-1","walk":"plan-1","phase":"ended","outcome":"stopped",` + `"touches":[{"machine":"novox","modules":["node-tools","messenger"],"carried":["messenger"]}],` + `"started_at":"2026-10-12T11:12:00Z","ended_at":"2026-10-12T11:19:00Z","updated":"2026-10-12T11:19:01Z"}`)) if err != nil || !ok || end.Outcome != WalkStopped || !end.At.Equal(at.Add(7*time.Minute)) || len(end.Touches[0].Modules) != 2 { t.Fatalf("%+v %v %v", end, ok, err) } if _, ok, err := DecodeWalk([]byte(`{"id":"plan-1","state":"walking","tier":1}`)); ok || err != nil { t.Fatalf("a move inside a walk read as its start or end: %v %v", ok, err) } if _, _, err := DecodeWalk([]byte(`{"walk":"plan-1","phase":"ended"}`)); err == nil { t.Fatalf("an end without its outcome was read") } w := Walk{} if w.bound() != DefaultWalkBound { t.Fatalf("a walk without a bound is bounded at %s", w.bound()) } // A start older than its bound is history: nothing is held. h, _, _, c, _ := newHolder(t) walk(t, h, "plan-old", WalkStarted, "", c.now().Add(-time.Hour), 1200, novox("mesh-tools")) if len(h.walks) != 0 { t.Fatalf("an old start opened a walk: %+v", h.walks) } } // Review of pull request 230, 1: no channel can send at a walk's end or bound. What it held is not lost with the // walk: it is unsent, and said on its own once a channel can send. func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) { for _, at := range []string{"end", "bound"} { t.Run(at, func(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-c", WalkStarted, "", c.now(), 600, novox("mesh-tools")) k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, 2*time.Minute) down := errors.New("no session") tg.notReady, dt.notReady = down, down if at == "end" { walk(t, h, "plan-c", WalkEnded, WalkFailed, c.now(), 0) runFor(h, c, time.Minute) } else { runFor(h, c, 10*time.Minute) } if saidOn(tg, dt) != 0 { t.Fatalf("said on a channel that cannot send: %+v %+v", tg.sends, dt.sends) } if r := h.open[k.Key]; r == nil || r.Pending == "" { t.Fatalf("what the walk held is not left unsent: %+v", r) } tg.notReady, dt.notReady = nil, nil runFor(h, c, 2*time.Minute) if !heard(headlineOf(k.Key), dt) { t.Fatalf("what the walk held was never said: %v", outcomes(h, k.Key)) } }) } } // Review of pull request 230, 2: a message held by walk A, about a machine walk B also touches, is held on at // A's end while B — started after it was held — is still open, and told at B's end. func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) k := cond("machine.novox.network", Urgent, "novox lost its default route") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools")) runFor(h, c, time.Minute) walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, time.Minute) if heard(headlineOf(k.Key), tg, dt) { t.Fatalf("told at the first walk's end while the second touches it: %+v %+v", tg.sends, dt.sends) } if got := h.heldBy(k.Key, c.now()); len(got) != 1 || got[0] != "plan-b" { t.Fatalf("the second walk does not hold it on: %v", got) } walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, time.Minute) if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) { t.Fatalf("the second walk's end did not tell it: %+v %+v", tg.sends, dt.sends) } } // Review of pull request 230, 3: a router back after more than a day reads back a walk whose end never came; // what it held is said, with the still-running line, before the walk is forgotten. func TestAWalkReadBackADayLateSaysWhatItHeldBeforeItIsForgotten(t *testing.T) { h, tg, dt, c, st := newHolder(t) walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("mesh-tools")) k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) c.pass(25 * time.Hour) again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf} if err := again.Load(); err != nil { t.Fatal(err) } again.Start() runFor(again, c, 2*time.Minute) if !heard("is still running", dt) || !heard("- Urgent: "+headlineOf(k.Key), dt) { t.Fatalf("what the walk held was forgotten unsaid: desk %+v, %v", dt.sends, outcomes(again, k.Key)) } if len(again.walks) != 0 { t.Fatalf("the walk was not forgotten: %+v", again.walks) } if ws, _ := st.Walks(); len(ws) != 0 { t.Fatalf("the walk is still kept: %+v", ws) } } // Review of pull request 230, 4 (and of mesh-controller pull request 247): a short walk's end can come before // its start. The start of a walk already ended opens nothing — also after the router's restart — while the // next walk under the same id, started after that end, does. func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) { h, tg, dt, c, st := newHolder(t) started := c.now() c.pass(20 * time.Second) end, _, err := DecodeWalk([]byte(`{"walk":"plan-e","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` + `"started_at":"` + started.Format(time.RFC3339Nano) + `","ended_at":"` + c.now().Format(time.RFC3339Nano) + `"}`)) if err != nil { t.Fatal(err) } h.Walked(end) c.pass(time.Second) walk(t, h, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools")) if len(h.walks) != 0 { t.Fatalf("a start heard after its end opened a walk: %+v", h.walks) } again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf} if err := again.Load(); err != nil { t.Fatal(err) } again.Start() walk(t, again, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools")) if len(again.walks) != 0 { t.Fatalf("after a restart, a start heard after its end opened a walk: %+v", again.walks) } k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() again.Condition(EventRaised, k) runFor(again, c, time.Minute) if !heard(headlineOf(k.Key), dt) { t.Fatalf("held by a walk that had ended: %v", outcomes(again, k.Key)) } // Retried: the same id walks again, started after the end. walk(t, again, "plan-e", WalkStarted, "", c.now(), 600, novox("mesh-tools")) if len(again.walks) != 1 { t.Fatalf("a walk started after the last end did not open: %+v", again.walks) } } // Review of pull request 230, non-blocking: a message whose answers became asked while a walk held it is never // held (ADR 0308 §3); when the desk's answers are lost, it is said again in place like any other, so its // buttons work again. func TestARestartSaysAgainAnAskedMessageAWalkTookBeforeItsAsk(t *testing.T) { h, _, dt, c, _, _ := newRouter(t) h.Start() stand(h, c) walk(t, h, "plan-f", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) k := cond("machine.novox.units", Urgent, "2 failed services on novox") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) ask := ackAsk(c, "units") ask.About = k.Key submit(t, h, ConditionOwner, ask) runFor(h, c, time.Minute) stand(h, c) if !heard(headlineOf(k.Key), dt) { t.Fatalf("the ask about it was not offered: %v", outcomes(h, k.Key)) } replaced := len(dt.replaced) c.pass(time.Minute) h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: c.now()}) runFor(h, c, time.Minute) if len(dt.replaced) != replaced+1 { t.Fatalf("its answers were not made to work again: replaced %+v, %v", dt.replaced[replaced:], outcomes(h, "(answers lost)")) } } // Review of pull request 230, non-blocking: a walk's end that cannot be said as it is (here a module named by a // file path) is said in its plain form, and what it held, still open, is said on its own, not marked told. func TestWhatAWalksPlainEndCannotCarryIsSaidOnItsOwn(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-g", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools")) k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) walk(t, h, "plan-g", WalkEnded, WalkFailed, c.now(), 0) runFor(h, c, time.Minute) if !heard("A delivery ended", dt) { t.Fatalf("the plain end was not said: %+v", dt.sends) } if !heard(headlineOf(k.Key), tg, dt) { t.Fatalf("what the walk held was marked told and never said: %v", outcomes(h, k.Key)) } } // Issue 538 A: what a walk held is marked told only once the message carrying it was sent. When that message // could not be sent and the router restarts before its retry, the message is gone with it: what it held is not // marked told, it stays owed in the state, and the restarted router says it — an urgent one and a warning alike, // once each, never left as history. func TestAHeldMessageIsToldOnlyOnceTheWalksMessageWasSent(t *testing.T) { for _, then := range []string{"sent", "restart urgent", "restart warning"} { t.Run(then, func(t *testing.T) { h, tg, dt, c, st := newHolder(t) sev := Urgent if then == "restart warning" { sev = Warning } walk(t, h, "plan-h", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) k := cond("machine.novox.tools-silent", sev, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) down := errors.New("cannot connect") tg.fail, dt.fail = down, down walk(t, h, "plan-h", WalkEnded, WalkFailed, c.now(), 0) runFor(h, c, time.Minute) if saidOn(tg, dt) != 0 { t.Fatalf("said on channels that cannot send: %+v %+v", tg.sends, dt.sends) } if r := h.open[k.Key]; r == nil || r.told() { t.Fatalf("marked told while the message carrying it was not sent: %+v", r) } tg.fail, dt.fail = nil, nil if then == "sent" { runFor(h, c, 2*time.Minute) if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) || saidOn(tg, dt) != 2 { t.Fatalf("not told once in the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends) } if r := h.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Sent["telegram"] != "" || r.Sent["desktop"] != "" { t.Fatalf("not marked told on both channels once sent: %+v", r) } runFor(h, c, 10*time.Minute) if saidOn(tg, dt) != 2 { t.Fatalf("said again after the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends) } return } again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf} if err := again.Load(); err != nil { t.Fatal(err) } again.Start() again.Sync([]Condition{k}, c.now()) runFor(again, c, 5*time.Minute) if !heard(headlineOf(k.Key), tg, dt) { t.Fatalf("what the lost message carried was not said after the restart: %v", outcomes(again, k.Key)) } said := saidOn(tg, dt) if r := again.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Pending != "" { t.Fatalf("said, and still owed or not marked told: %+v", r) } runFor(again, c, 20*time.Minute) // within its reminder's bound: nothing more is due again.Sync([]Condition{k}, c.now()) runFor(again, c, time.Minute) if saidOn(tg, dt) != said { t.Fatalf("said again: telegram %+v, desk %+v", tg.sends, dt.sends) } }) } } // Issue 538 B: at a walk's end, a later walk that touches what it held takes over only what a first hold would // take: not a message silenced meanwhile, nor one an ask waiting for the operator is about (ADR 0308 §3). func TestAWalksEndHandsOnOnlyWhatALaterWalkWouldHold(t *testing.T) { h, tg, dt, c, _, _ := newRouter(t) h.Start() stand(h, c) walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) quiet := cond("machine.novox.network", Urgent, "novox lost its default route") quiet.Raised = c.now() h.Condition(EventRaised, quiet) asked := cond("machine.novox.units", Urgent, "2 failed services on novox") asked.Raised = c.now() h.Condition(EventRaised, asked) runFor(h, c, time.Minute) stand(h, c) if heard(headlineOf(quiet.Key), tg, dt) || heard(headlineOf(asked.Key), tg, dt) { t.Fatalf("the test's messages were not held") } quiet.SilencedTill, quiet.SilencedWhy, quiet.At = c.now().Add(7*24*time.Hour), "known", c.now() h.Condition(EventChanged, quiet) ask := ackAsk(c, "units") ask.About = asked.Key submit(t, h, ConditionOwner, ask) runFor(h, c, time.Minute) stand(h, c) walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools")) runFor(h, c, time.Minute) stand(h, c) helds := func(key string) int { n := 0 for _, o := range outcomes(h, key) { if o == "held" { n++ } } return n } before := map[string]int{quiet.Key: helds(quiet.Key), asked.Key: helds(asked.Key)} walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, time.Minute) for _, key := range []string{quiet.Key, asked.Key} { if got := h.heldBy(key, c.now()); len(got) != 0 { t.Errorf("%s was handed on to %v, which a first hold would never hold", key, got) } if helds(key) != before[key] { t.Errorf("the history says %s held again: %v", key, outcomes(h, key)) } } } // Issue 538 C: the controller's clock ahead of the router's. A short walk's end, heard before its start, opens // nothing when its start comes, whether the end said the walk's start or only its own moment; the next walk // under its id, started after that end, still opens. func TestAStartHeardAfterItsEndOpensNothingWhenTheControllersClockIsAhead(t *testing.T) { for _, said := range []string{"both", "end only"} { t.Run(said, func(t *testing.T) { h, _, _, c, _ := newHolder(t) ended := c.now().Add(10 * time.Minute) // by the controller's clock started := ended.Add(-20 * time.Second) body := `{"walk":"plan-k","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` + `"ended_at":"` + ended.Format(time.RFC3339Nano) + `"` if said == "both" { body += `,"started_at":"` + started.Format(time.RFC3339Nano) + `"` } end, _, err := DecodeWalk([]byte(body + `}`)) if err != nil { t.Fatal(err) } h.Walked(end) c.pass(time.Second) walk(t, h, "plan-k", WalkStarted, "", started, 600, novox("mesh-tools")) if len(h.walks) != 0 { t.Fatalf("a start heard after its end opened a walk: %+v", h.walks) } walk(t, h, "plan-k", WalkStarted, "", ended.Add(time.Minute), 600, novox("mesh-tools")) if len(h.walks) != 1 { t.Fatalf("the next walk under its id, started after the end, did not open: %+v", h.walks) } }) } }