package main import ( "os" "path/filepath" "strings" "testing" "time" ) // The tests of novox/hq ADR 0505 "How it is checked" for the router: what a delivery changes is held until it has // settled, every problem message says whether a delivery ran, and a failed delivery is told at once. // ADR 0505 §3: the settle is a setting, settle-seconds, two minutes when none is given. func TestTheSettleIsReadFromTheSettingsFileAndIsTwoMinutesByDefault(t *testing.T) { read := func(body string) Settings { p := filepath.Join(t.TempDir(), "settings.json") if err := os.WriteFile(p, []byte(body), 0o600); err != nil { t.Fatal(err) } s, err := readSettings(p) if err != nil { t.Fatal(err) } return s } if s := read(`{"settle-seconds": 60}`); s.SettleSeconds == nil || *s.SettleSeconds != 60 { t.Fatalf("set: %+v", s) } if s := read(`{"time-zone": ""}`); s.SettleSeconds != nil { t.Fatalf("unset: %+v", s) } h := &Holder{} if got := h.settle(); got != 2*time.Minute { t.Fatalf("no setting: settle %s, want 2m", got) } h.Settle = func() time.Duration { return time.Minute } if got := h.settle(); got != time.Minute { t.Fatalf("a setting of 60 s: settle %s, want 1m", got) } h.Settle = func() time.Duration { return 0 } if got := h.settle(); got != 0 { t.Fatalf("a setting of 0: settle %s, want none", got) } } // shanksDeclared is a touch as the controller says a machine whose declaration the walk's merges change and to which // it sends nothing (mesh-controller walk_said.go walkTouch, `declared`, ADR 0505 §2b). var shanksDeclared = Touch{Machine: "shanks", Modules: []string{"mesh-build-image"}, Declared: true} // ADR 0505 §2b: a machine whose declaration a walk's merges change is held like a machine it sends to, and is not // named among the machines the walk reached. func TestAMachineWhoseDeclarationTheWalkChangesIsHeldAndNotNamedAsReached(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared) k := cond("machine.shanks.uncomposable", Urgent, "shanks cannot get new instructions") k.Machine, k.Raised = "shanks", c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) if !has(outcomes(h, k.Key), "held") || heard(headlineOf(k.Key), tg, dt) { t.Fatalf("a condition on the declared machine was not held: %v", outcomes(h, k.Key)) } if got := touchWords(mergeTouches([]Touch{novox("gitea"), shanksDeclared}, nil)); got != "gitea on novox" { t.Fatalf("what the walk reached reads %q, want only novox", got) } if got := mergeTouches([]Touch{shanksDeclared}, []Touch{{Machine: "shanks", Modules: []string{"buildah"}}}); got[0].Declared { t.Fatalf("a machine the walk also sends to stayed declared: %+v", got) } } // ADR 0505 §3: a condition raised 30 seconds after a delivered walk's end, on what it touched, is held; at the end // plus the settle the end message tells it under "after the delivery of …" when it is still open, and only counts // it when it cleared. func TestAConditionRaisedInTheSettleIsHeldAndToldAtItsEnd(t *testing.T) { for _, clears := range []bool{false, true} { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-s", WalkStarted, "", c.now(), 600, novox("gitea")) runFor(h, c, 3*time.Minute) walk(t, h, "plan-s", WalkEnded, WalkDelivered, c.now(), 0, novox("gitea")) c.pass(30 * time.Second) k := cond("machine.novox.uncomposable", Urgent, "novox cannot get new instructions") k.Machine, k.Raised = "novox", c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) if heard(headlineOf(k.Key), tg, dt) || !has(outcomes(h, k.Key), "held") { t.Fatalf("raised in the settle and not held: %v", outcomes(h, k.Key)) } if heard("Delivered", tg, dt) { t.Fatalf("the end was told before the settle passed: %+v", dt.sends) } if clears { h.Condition(EventCleared, Condition{Key: k.Key, At: c.now()}) } runFor(h, c, time.Minute) switch { case !clears && !heard("Still open after the delivery of gitea to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt): t.Fatalf("still open at the settle's end and not told under the delivery: %+v", dt.sends) case clears && (heard(headlineOf(k.Key), tg, dt) || !heard("1 problem it raised resolved during it.", dt)): t.Fatalf("cleared in the settle: told, or not counted: %+v", dt.sends) } } } // ADR 0505 §3: a reminder of an older condition that falls on a touched subject while a walk is open is held like // any other message, and told at the settle's end only when the condition is still open. func TestAReminderDuringAWalkIsHeld(t *testing.T) { h, tg, dt, c, _ := newHolder(t) k := cond("machine.novox.network", Urgent, "novox lost its default route") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, time.Minute) first := saidOn(tg, dt) if first == 0 { t.Fatalf("the condition was not said before the walk") } runFor(h, c, RemindUrgent-3*time.Minute) walk(t, h, "plan-r", WalkStarted, "", c.now(), 1200, novox("node-tools")) runFor(h, c, 5*time.Minute) if saidOn(tg, dt) != first { t.Fatalf("the reminder was said during the walk: %+v", dt.sends) } if !has(outcomes(h, k.Key), "held") { t.Fatalf("the reminder was not held: %v", outcomes(h, k.Key)) } walk(t, h, "plan-r", WalkEnded, WalkDelivered, c.now(), 0) runFor(h, c, DefaultSettle+time.Minute) if !heard("Still open after the delivery of node-tools to novox:", dt) { t.Fatalf("the held reminder was not told at the settle's end: %+v", dt.sends) } } // ADR 0505 §3: a router that starts again while a walk settles holds on and tells its end once the settle passed. func TestAWalkThatSettlesSurvivesARouterRestart(t *testing.T) { h, tg, dt, c, st := newHolder(t) walk(t, h, "plan-x", WalkStarted, "", c.now(), 1200, novox("mesh-tools")) runFor(h, c, time.Minute) walk(t, h, "plan-x", WalkEnded, WalkDelivered, c.now(), 0) c.pass(10 * time.Second) again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf} if err := again.Load(); err != nil { t.Fatal(err) } again.Start() k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() again.Condition(EventRaised, k) runFor(again, c, time.Minute) if heard(headlineOf(k.Key), tg, dt) { t.Fatalf("not held by the settling walk read back: %v", outcomes(again, k.Key)) } runFor(again, c, DefaultSettle) if !heard("Still open after the delivery of mesh-tools to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) { t.Fatalf("the settled walk's end was not told after the restart: %+v", dt.sends) } walk(t, again, "plan-x", WalkStarted, "", c.now().Add(-5*time.Minute), 1200, novox("mesh-tools")) if len(again.walks) != 0 { t.Fatalf("its late start opened it again: %+v", again.walks) } } // ADR 0505 §6: a condition raised on a touched subject 30 seconds after a failed walk is sent at once, even while // another open walk touches the same machine; the same condition after a delivered walk is held. func TestAConditionJustAfterAFailedWalkIsSentAtOnceAndAfterADeliveredOneHeld(t *testing.T) { for _, outcome := range []string{WalkFailed, WalkStopped, WalkDelivered} { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-f", WalkStarted, "", c.now(), 600, novox("gitea")) walk(t, h, "plan-o", WalkStarted, "", c.now(), 1200, novox("mesh-issues")) runFor(h, c, 2*time.Minute) walk(t, h, "plan-f", WalkEnded, outcome, c.now(), 0, novox("gitea")) runFor(h, c, 30*time.Second) k := cond("machine.novox.uncomposable", Urgent, "novox cannot get new instructions") k.Machine, k.Raised = "novox", c.now() h.Condition(EventRaised, k) runFor(h, c, 30*time.Second) sent := heard(headlineOf(k.Key), tg, dt) if outcome != WalkDelivered && !sent { t.Fatalf("after a walk that %s, held: %v", outcome, outcomes(h, k.Key)) } if outcome == WalkDelivered && sent { t.Fatalf("after a delivered walk, sent at once: %+v", dt.sends) } } } // ADR 0505 §6: an urgent condition on a machine no open or settling walk touches is sent at once during a walk. func TestAnUrgentConditionOnAnUntouchedMachineIsSentAtOnceDuringAWalk(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-u", WalkStarted, "", c.now(), 1200, novox("gitea")) k := cond("machine.ace.silent", Urgent, "ace has not been heard") k.Machine, k.Raised = "ace", c.now() h.Condition(EventRaised, k) if !heard(headlineOf(k.Key), tg, dt) { t.Fatalf("held, while no walk touches ace: %v", outcomes(h, k.Key)) } } // bodyOf is a held or open record's message as it would be sent now. func bodyOf(t *testing.T, h *Holder, key, what string) string { t.Helper() h.mu.Lock() r := h.open[key] h.mu.Unlock() if r == nil { t.Fatalf("%s is not open", key) } return h.say(r, what).Body } // ADR 0505 §1: a problem's message ends, after its since-when line, with the delivery line: during a walk, in a // walk's settle, after a failed walk, and with no walk. func TestEveryProblemMessageEndsWithWhetherADeliveryRan(t *testing.T) { h, tg, dt, c, _ := newHolder(t) k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) if !strings.HasSuffix(tg.sends[0].Body, ".\nNo delivery running.") || !strings.Contains(tg.sends[0].Body, "\nSince ") { t.Fatalf("no walk: %q", tg.sends[0].Body) } walk(t, h, "plan-l", WalkStarted, "", c.now(), 600, novox("gitea", "mesh-issues"), shanksDeclared) if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, ".\nDuring the delivery of gitea, mesh-issues to novox.") { t.Fatalf("during a walk: %q", b) } runFor(h, c, time.Minute) walk(t, h, "plan-l", WalkEnded, WalkDelivered, c.now(), 0) c.pass(40 * time.Second) if b := bodyOf(t, h, k.Key, "raised"); !strings.HasSuffix(b, "\n40 s after the delivery of gitea, mesh-issues to novox, which succeeded.") { t.Fatalf("in the settle of a delivered walk: %q", b) } runFor(h, c, 3*time.Minute) if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, "\nNo delivery running.") { t.Fatalf("after the settle: %q", b) } walk(t, h, "plan-m", WalkStarted, "", c.now(), 600, novox("gitea")) walk(t, h, "plan-m", WalkEnded, WalkFailed, c.now(), 0) c.pass(30 * time.Second) if b := bodyOf(t, h, k.Key, "reminder"); !strings.HasSuffix(b, "\n30 s after the delivery of gitea to novox, which failed.") { t.Fatalf("after a failed walk: %q", b) } keyless(t, dt.sends...) // Not on a clearing, a silencing or a plain notice. for _, what := range []string{"cleared", "silenced", "tell"} { if carriesDeliveryLine(&Record{Origin: "condition"}, what) { t.Errorf("a %s message carries the delivery line", what) } } if carriesDeliveryLine(&Record{Origin: "ask"}, "raised") { t.Errorf("an ask's own message carries the delivery line") } } // ADR 0505 §1: the line names at most six modules and counts the rest, and says no name the content rule refuses. func TestTheDeliveryLineNamesAtMostSixModulesAndNothingTheRuleRefuses(t *testing.T) { many := novox("a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8") if got := deliveryWords([]Touch{many}); got != "a1, a2, a3, a4, a5, a6 and 2 more to novox" { t.Fatalf("eight modules: %q", got) } h, _, _, c, _ := newHolder(t) walk(t, h, "plan-p", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools")) k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent") k.Raised = c.now() h.Condition(EventRaised, k) if b := bodyOf(t, h, k.Key, "raised"); !strings.HasSuffix(b, "\nDuring a delivery.") { t.Fatalf("a refused name in the line: %q", b) } } // ADR 0505 §7: a condition on a machine whose declaration three back-to-back walks change (a machine that stays // behind is a declared touch of each) reaches the operator at the first walk's bound and settle, while the third // still runs: never held walk after walk. func TestADeclaredMachinesConditionAcrossThreeWalksIsSaidAtTheFirstWalksLimit(t *testing.T) { h, tg, dt, c, _ := newHolder(t) t0 := c.now() walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared) // limit: t0 + 10 + 2 min k := cond("machine.shanks.uncomposable", Urgent, "shanks cannot get new instructions") k.Machine, k.Raised = "shanks", c.now() h.Condition(EventRaised, k) runFor(h, c, 5*time.Minute) walk(t, h, "plan-1", WalkEnded, WalkDelivered, c.now(), 0, novox("gitea"), shanksDeclared) runFor(h, c, time.Minute) walk(t, h, "plan-2", WalkStarted, "", c.now(), 600, novox("mesh-issues"), shanksDeclared) runFor(h, c, 3*time.Minute) walk(t, h, "plan-2", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-issues"), shanksDeclared) runFor(h, c, time.Minute) walk(t, h, "plan-3", WalkStarted, "", c.now(), 1200, novox("gitea"), shanksDeclared) for c.now().Before(t0.Add(12*time.Minute - 10*time.Second)) { runFor(h, c, 5*time.Second) } if heard(headlineOf(k.Key), tg, dt) { t.Fatalf("said before the first walk's limit: %+v", dt.sends) } runFor(h, c, 30*time.Second) if !heard(headlineOf(k.Key), tg, dt) { t.Fatalf("held past the first walk's bound and settle: %v", outcomes(h, k.Key)) } } // ADR 0505 §2: a walk's start or end said again by the controller with its declared machines (declared_read) acts // once: one walk with both sayings' touches, and one end message. func TestAWalkSaidAgainWithItsDeclaredMachinesActsOnce(t *testing.T) { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, novox("gitea")) walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared) if len(h.walks) != 1 || len(h.walks["plan-r"].Touches) != 2 { t.Fatalf("said again it is not one walk with both touches: %+v", h.walks) } runFor(h, c, time.Minute) walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, novox("gitea")) walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, novox("gitea"), shanksDeclared) runFor(h, c, 3*time.Minute) n := 0 for _, m := range append(append([]Message(nil), dt.sends...), tg.sends...) { if strings.HasPrefix(m.Title, "Delivery failed") { n++ } } if n != 1 { t.Fatalf("its end was told %d times: %+v", n, dt.sends) } } // ADR 0505 §7: a message two walks first held, the sooner of which ends at its limit, is said exactly once — in that // walk's end message — and not again on its own for the other walk; once per channel when it is urgent. func TestAMessageTwoWalksFirstHeldIsSaidOnceAtTheSoonerLimit(t *testing.T) { for _, sev := range []string{Warning, Urgent} { h, tg, dt, c, _ := newHolder(t) walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, novox("gitea")) // its limit: 6 min and the settle walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("mesh-issues")) // its limit: 22 min k := cond("machine.novox.network", sev, "novox lost its default route") k.Raised = c.now() h.Condition(EventRaised, k) runFor(h, c, 7*time.Minute) walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0) // settles until its limit, 8 min after the hold runFor(h, c, 3*time.Minute) count := func(ch *fakeChannel) int { n := 0 for _, m := range ch.sends { if strings.Contains(m.Text(), headlineOf(k.Key)) { n++ } } return n } want := map[*fakeChannel]int{dt: 1, tg: 0} if sev == Urgent { want[tg] = 1 } for ch, n := range want { if got := count(ch); got != n { t.Fatalf("%s: said %d times on %s, want %d: %+v", sev, got, ch.name, n, ch.sends) } } if !heard("Still open after the delivery of gitea to novox:", dt) { t.Fatalf("%s: not told in the sooner walk's end message: %+v", sev, dt.sends) } } }