772 lines
32 KiB
Go
772 lines
32 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The tests of novox/hq ADR 0308 "How it is checked" §2–§6, with the controller's walk events given by a fake
|
|
// source: the body the controller's plan-moved carries at a walk's start and end (ADR 0308 §1).
|
|
|
|
// walkBody is a plan-moved body as the controller says a walk's start or end (mesh-controller pull request 247).
|
|
func walkBody(t *testing.T, id, phase, outcome string, at time.Time, bound int, touches ...Touch) []byte {
|
|
t.Helper()
|
|
b := map[string]any{"id": id, "state": "walking", "walk": id, "phase": phase, "touches": touches,
|
|
"updated": at.Format(time.RFC3339Nano), "budget_class": "leaf"}
|
|
if phase == WalkStarted {
|
|
b["started_at"] = at.Format(time.RFC3339Nano)
|
|
} else {
|
|
b["ended_at"] = at.Format(time.RFC3339Nano)
|
|
}
|
|
if outcome != "" {
|
|
b["outcome"] = outcome
|
|
}
|
|
if bound > 0 {
|
|
b["bound_seconds"] = bound
|
|
}
|
|
raw, err := json.Marshal(b)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// walk says a walk's start or end to the router, through the decoder the bus's events go through.
|
|
func walk(t *testing.T, h *Holder, id, phase, outcome string, at time.Time, bound int, touches ...Touch) {
|
|
t.Helper()
|
|
ev, ok, err := DecodeWalk(walkBody(t, id, phase, outcome, at, bound, touches...))
|
|
if err != nil || !ok {
|
|
t.Fatalf("the walk event was not read: %v %v", ok, err)
|
|
}
|
|
h.Walked(ev)
|
|
}
|
|
|
|
func novox(modules ...string) Touch { return Touch{Machine: "novox", Modules: modules} }
|
|
|
|
// heard says whether any message on the channels carries the words: sent, edited or said again in place.
|
|
func heard(words string, chs ...*fakeChannel) bool {
|
|
for _, ch := range chs {
|
|
for _, m := range ch.sends {
|
|
if strings.Contains(m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
for _, m := range ch.edits {
|
|
if strings.Contains(m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
for _, r := range ch.replaced {
|
|
if strings.Contains(r.m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// outcomes are the history's outcomes for a key, oldest first.
|
|
func outcomes(h *Holder, key string) []string {
|
|
var out []string
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
for _, s := range h.recent {
|
|
if s.Key == key {
|
|
out = append(out, s.Outcome)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func has(list []string, want string) bool {
|
|
for _, s := range list {
|
|
if s == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// §2: during a walk, a message about a machine or a module it touches is held, not sent, and the channel's
|
|
// history says it was held — a module's condition, a machine's, and a module's notice alike.
|
|
func TestDuringAWalkWhatItTouchesIsHeldAndTheHistorySaysSo(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("mesh-tools", "node-tools"))
|
|
mod := cond("module.node-tools.novox.unhealthy", Urgent, "the tool runner on novox does not answer")
|
|
mod.Machine, mod.Raised = "novox", c.now()
|
|
h.Condition(EventRaised, mod)
|
|
machine := cond("machine.novox.claude-code.agent-home.data-shrank", Urgent, "the agent's home shrank")
|
|
machine.Raised = c.now()
|
|
h.Condition(EventRaised, machine)
|
|
if _, err := h.Notify("tools.slow", Urgent, "tools answer slowly", "node-tools", false, "tools slow",
|
|
"Needs you: look at the tools."); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runFor(h, c, 2*time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("held messages were said: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
for _, key := range []string{mod.Key, machine.Key, "notify.tools.slow"} {
|
|
if !has(outcomes(h, key), "held") {
|
|
t.Errorf("the history does not say %s was held: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
if len(h.Open()) != 3 {
|
|
t.Fatalf("held messages are not kept open: %+v", h.Open())
|
|
}
|
|
}
|
|
|
|
// §2: a message touched by two overlapping walks is held until the second of them ends; the first's end says
|
|
// nothing of it.
|
|
func TestAMessageTouchedByTwoOverlappingWalksIsHeldUntilTheSecondEnds(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
c.pass(time.Minute)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"), Touch{Machine: "ace", Modules: []string{"node-tools"}})
|
|
k := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-tools"))
|
|
runFor(h, c, DefaultSettle+time.Minute) // its end is told once it settled (ADR 0505 §3)
|
|
if heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("said at the first walk's end, while the second still holds it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if !heard("Delivered: mesh-tools on novox", tg, dt) {
|
|
t.Fatalf("the first walk's end was not told: %+v", dt.sends)
|
|
}
|
|
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, DefaultSettle+time.Minute)
|
|
if !heard("Still open after the delivery of node-tools to ace, novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("the second walk's end did not tell it: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
}
|
|
|
|
// §3: an ask, a secret given, a message about a machine no open walk touches and a walk's own failure are
|
|
// sent at once, while a walk touches the machine they are on.
|
|
func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
walk(t, h, "plan-9", WalkStarted, "", c.now(), 1200, novox("telegram", "mesh-tools"))
|
|
for _, key := range []string{
|
|
"machine.novox.telegram.telegram-token.secret-given", // a secret given, on a touched machine
|
|
"machine.novox.mounts.smb.secret-replaced", // a secret replaced
|
|
"machine.ace.silent", // a machine no walk touches
|
|
"delivery.leaf.over-budget", // the walk's own delivery
|
|
"plan.plan-9.failed", // the walk itself
|
|
} {
|
|
k := cond(key, Urgent, "it happened")
|
|
k.Raised = c.now()
|
|
if !strings.HasPrefix(key, "machine.") {
|
|
k.Machine = "novox" // a walk's own failure, on a machine it touches
|
|
}
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if !heard(headlineOf(key), tg, dt) {
|
|
t.Errorf("%s was not sent at once during the walk: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
// An ask's own message, and an ask about a held condition: both offered at once.
|
|
submit(t, h, ConditionOwner, ackAsk(c, "own"))
|
|
runFor(h, c, time.Minute)
|
|
if !has(outcomes(h, "ask."+ConditionOwner+".own"), "sent") {
|
|
t.Errorf("an ask waiting for the operator was held: %v", outcomes(h, "ask."+ConditionOwner+".own"))
|
|
}
|
|
held := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
held.Raised = c.now()
|
|
h.Condition(EventRaised, held)
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(held.Key), tg, dt) {
|
|
t.Fatalf("a condition about a touched machine was not held")
|
|
}
|
|
about := ackAsk(c, "about")
|
|
about.About = held.Key
|
|
submit(t, h, ConditionOwner, about)
|
|
runFor(h, c, time.Minute)
|
|
if !heard(headlineOf(held.Key), dt) {
|
|
t.Fatalf("the ask about a held condition was held: %v", outcomes(h, held.Key))
|
|
}
|
|
}
|
|
|
|
// §4: a walk's end is one message — its outcome, what it reached and how long it took — then each held message
|
|
// still open, one line each; the one that cleared is only counted, and nothing more is said of them after.
|
|
func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-3", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
still := cond("machine.novox.tools-silent", Warning, "the tools on novox are silent")
|
|
still.Raised = c.now()
|
|
h.Condition(EventRaised, still)
|
|
gone := cond("module.mesh-tools.novox.unhealthy", Warning, "mesh-tools is unhealthy on novox")
|
|
gone.Raised = c.now()
|
|
h.Condition(EventRaised, gone)
|
|
c.pass(time.Second)
|
|
other := cond("machine.novox.core-behind", Urgent, "novox runs an older core")
|
|
other.Raised = c.now()
|
|
h.Condition(EventRaised, other)
|
|
runFor(h, c, 2*time.Minute)
|
|
h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()})
|
|
runFor(h, c, 5*time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said during the walk: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
walk(t, h, "plan-3", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
// Urgent among what it holds: on every kind, one message each.
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 {
|
|
t.Fatalf("the end was not one message per kind: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
m := dt.sends[0]
|
|
if m.Title != "Delivery failed: mesh-tools on novox" || !strings.Contains(m.Body, "It took 7 min.") {
|
|
t.Fatalf("the end does not say its outcome, what it reached and how long: %q", m.Text())
|
|
}
|
|
want := "Still open after the delivery of mesh-tools to novox:\n- " + headlineOf(still.Key) + "\n- Urgent: " + headlineOf(other.Key) +
|
|
"\n1 problem it raised resolved during it."
|
|
if !strings.Contains(m.Body, want) || strings.Contains(m.Text(), headlineOf(gone.Key)) {
|
|
t.Fatalf("the end message %q does not say %q", m.Body, want)
|
|
}
|
|
keyless(t, m)
|
|
runFor(h, c, 20*time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 || len(dt.replaced)+len(tg.replaced) != 0 {
|
|
t.Fatalf("what the end told was said again: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
}
|
|
|
|
// §4: a walk that delivered with nothing held and nothing open is one line, and nothing at all on a kind set
|
|
// to say only what needs the operator.
|
|
func TestADeliveredWalkWithNothingHeldIsOneLineOrNothing(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-4", WalkStarted, "", c.now(), 600, novox("mesh-tools"), Touch{Machine: "ace", Modules: []string{"mesh-tools"}})
|
|
runFor(h, c, 3*time.Minute)
|
|
walk(t, h, "plan-4", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, DefaultSettle+time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
if m := dt.sends[0]; m.Title != "Delivered: mesh-tools on ace, novox, in 3 min" || m.Body != "" {
|
|
t.Fatalf("not one line: %q", m.Text())
|
|
}
|
|
h.NeedsOnly = func(kind string) bool { return kind == "desktop" }
|
|
walk(t, h, "plan-5", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
runFor(h, c, 3*time.Minute)
|
|
walk(t, h, "plan-5", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, DefaultSettle+time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("told on a kind that says only what needs the operator: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
if !has(outcomes(h, "walk.plan-5"), "quiet: this kind says only what needs the operator") {
|
|
t.Fatalf("the history does not say why nothing was told: %v", outcomes(h, "walk.plan-5"))
|
|
}
|
|
}
|
|
|
|
// §5: the bound is the controller's (bound_seconds: twice the largest budget of the walk's deliveries; none
|
|
// given, 15 minutes), and the settle is added to it (ADR 0505 §7). At it, what is held and still open is sent
|
|
// with the still-running line, and what cleared is counted; before it, nothing; after it, the walk holds nothing
|
|
// more.
|
|
func TestAHoldEndsAtTheWalksBound(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
given int
|
|
bound time.Duration
|
|
}{
|
|
{"core: twice ten minutes", 1200, 20 * time.Minute},
|
|
{"leaf at five minutes", 600, 10 * time.Minute},
|
|
{"leaf at three minutes", 360, 6 * time.Minute},
|
|
{"no budget", 0, 15 * time.Minute},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
started := c.now()
|
|
walk(t, h, "plan-7", WalkStarted, "", started, tc.given, novox("node-tools"))
|
|
open := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
open.Raised = c.now()
|
|
h.Condition(EventRaised, open)
|
|
gone := cond("module.node-tools.novox.unhealthy", Urgent, "node-tools is unhealthy")
|
|
gone.Raised = c.now()
|
|
h.Condition(EventRaised, gone)
|
|
c.pass(30 * time.Second)
|
|
h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()})
|
|
runFor(h, c, tc.bound+DefaultSettle-time.Minute-30*time.Second)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said before the bound: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
runFor(h, c, 2*time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 {
|
|
t.Fatalf("at the bound: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
m := dt.sends[0]
|
|
since := started.In(h.zone()).Format("15:04")
|
|
for _, want := range []string{"is still running, since " + since + ".", "- Urgent: " + headlineOf(open.Key),
|
|
"1 problem it raised resolved during it."} {
|
|
if !strings.Contains(m.Body, want) {
|
|
t.Fatalf("the bound's message %q does not say %q", m.Text(), want)
|
|
}
|
|
}
|
|
// Past its bound the walk holds nothing: a new message about what it touches goes at once.
|
|
late := cond("machine.novox.disk", Urgent, "a disk fails on novox")
|
|
late.Raised = c.now()
|
|
h.Condition(EventRaised, late)
|
|
runFor(h, c, time.Minute)
|
|
if !heard(headlineOf(late.Key), dt) {
|
|
t.Fatalf("held past the bound: %v", outcomes(h, late.Key))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §2, kept: what a walk holds survives the router's restart, and is told at the walk's end.
|
|
func TestHeldMessagesSurviveARouterRestart(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
walk(t, h, "plan-8", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
c.pass(2 * time.Minute)
|
|
walk(t, again, "plan-8", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(again, c, DefaultSettle+time.Minute)
|
|
if !heard("- Urgent: "+headlineOf(k.Key), dt) || saidOn(tg, dt) != 2 {
|
|
t.Fatalf("what was held before the restart was not told at the walk's end: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
}
|
|
|
|
// §6, issue 531: the desk's notifiers on two machines start again a second apart, with four silenced messages
|
|
// shown there — one of them withheld for its content, a file path — and a fifth open, asked there, and
|
|
// withheld. Nothing is said again: no silenced, no withheld message; the pass runs once; and a router that
|
|
// starts again reads it back from its state and makes it no more.
|
|
func TestARestartSaysNothingItHasSaid(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
c.pass(2 * time.Minute)
|
|
keys := []string{"machine.novox.telegram.telegram-token.secret-given", "machine.novox.claude-code.agent-home.data-shrank",
|
|
"machine.shanks.mounts.smb-password-recalbox.secret-given", "machine.ace.nzbgeek.api-key.secret-given"}
|
|
var silenced []Condition
|
|
for i, key := range keys {
|
|
k := cond(key, Urgent, "it happened")
|
|
if i == 1 {
|
|
k.Explanation = "Needs you: look at /var/lib/agent-home, which shrank."
|
|
}
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
silenced = append(silenced, k)
|
|
}
|
|
withheldOpen := cond("machine.novox.units", Urgent, "2 failed services")
|
|
withheldOpen.Explanation = "Needs you: read /etc/systemd/system/x.service."
|
|
withheldOpen.Raised = c.now()
|
|
h.Condition(EventRaised, withheldOpen)
|
|
runFor(h, c, time.Minute)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = withheldOpen.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
for _, k := range silenced {
|
|
k.SilencedTill, k.SilencedWhy, k.At = c.now().Add(7*24*time.Hour), "known", c.now()
|
|
h.Condition(EventChanged, k)
|
|
}
|
|
runFor(h, c, time.Minute)
|
|
for _, key := range []string{silenced[1].Key, withheldOpen.Key} {
|
|
if r := h.open[key]; r == nil || r.Refused == "" {
|
|
t.Fatalf("the test's %s was not withheld for its content: %+v", key, r)
|
|
}
|
|
}
|
|
sends, edits, replaced, tgSaid := len(dt.sends), len(dt.edits), len(dt.replaced), saidOn(tg)
|
|
dt.edits = map[string]Message{}
|
|
passes := func(h *Holder) int { return len(outcomes(h, "(answers lost)")) }
|
|
// The notifiers on two machines started again a second apart: two standings, two times.
|
|
c.pass(time.Minute)
|
|
first := c.now()
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: first})
|
|
c.pass(time.Second)
|
|
h.Flush() // the runtime's tick, between the two standings
|
|
second := c.now()
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second})
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid {
|
|
t.Fatalf("the restart said again: sends %+v, edits %+v (was %d), replaced %+v", dt.sends[sends:], dt.edits, edits,
|
|
dt.replaced[replaced:])
|
|
}
|
|
if passes(h) != 1 {
|
|
t.Fatalf("the pass ran %d times for one restart", passes(h))
|
|
}
|
|
// The router starts again; the desk repeats its standing.
|
|
again := &Holder{Away: tg, Desk: dt, Store: h.Store, Asks: h.Asks, Bench: h.Bench, Publish: h.Publish, Now: c.now,
|
|
Logf: t.Logf, Zone: h.Zone, RootFree: h.RootFree, Node: h.Node}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
again.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second})
|
|
runFor(again, c, 2*time.Minute)
|
|
if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid || passes(again) != 1 {
|
|
t.Fatalf("the router's restart made the pass again: %d passes, sends %+v, edits %+v", passes(again),
|
|
dt.sends[sends:], dt.edits)
|
|
}
|
|
}
|
|
|
|
// The controller's walk events are read as ADR 0308 §1 says them; any other plan-moved is not a walk's.
|
|
func TestTheWalkEventsAreReadAsTheControllerSaysThem(t *testing.T) {
|
|
at := time.Date(2026, 10, 12, 11, 12, 0, 0, time.UTC)
|
|
ev, ok, err := DecodeWalk(walkBody(t, "plan-1", WalkStarted, "", at, 1200, novox("mesh-tools", "node-tools")))
|
|
if err != nil || !ok || ev.Walk != "plan-1" || ev.BoundSeconds != 1200 || ev.Class != "leaf" || !ev.At.Equal(at) ||
|
|
len(ev.Touches) != 1 || ev.Touches[0].Machine != "novox" || len(ev.Touches[0].Modules) != 2 {
|
|
t.Fatalf("%+v %v %v", ev, ok, err)
|
|
}
|
|
// As mesh-controller's walk events (its pull request 247) say an end: ended_at, and carried beside modules.
|
|
end, ok, err := DecodeWalk([]byte(`{"id":"plan-1","walk":"plan-1","phase":"ended","outcome":"stopped",` +
|
|
`"touches":[{"machine":"novox","modules":["node-tools","messenger"],"carried":["messenger"]}],` +
|
|
`"started_at":"2026-10-12T11:12:00Z","ended_at":"2026-10-12T11:19:00Z","updated":"2026-10-12T11:19:01Z"}`))
|
|
if err != nil || !ok || end.Outcome != WalkStopped || !end.At.Equal(at.Add(7*time.Minute)) || len(end.Touches[0].Modules) != 2 {
|
|
t.Fatalf("%+v %v %v", end, ok, err)
|
|
}
|
|
if _, ok, err := DecodeWalk([]byte(`{"id":"plan-1","state":"walking","tier":1}`)); ok || err != nil {
|
|
t.Fatalf("a move inside a walk read as its start or end: %v %v", ok, err)
|
|
}
|
|
if _, _, err := DecodeWalk([]byte(`{"walk":"plan-1","phase":"ended"}`)); err == nil {
|
|
t.Fatalf("an end without its outcome was read")
|
|
}
|
|
w := Walk{}
|
|
if w.bound() != DefaultWalkBound {
|
|
t.Fatalf("a walk without a bound is bounded at %s", w.bound())
|
|
}
|
|
// A start older than its bound is history: nothing is held.
|
|
h, _, _, c, _ := newHolder(t)
|
|
walk(t, h, "plan-old", WalkStarted, "", c.now().Add(-time.Hour), 1200, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("an old start opened a walk: %+v", h.walks)
|
|
}
|
|
|
|
}
|
|
|
|
// Review of pull request 230, 1: no channel can send at a walk's end or bound. What it held is not lost with the
|
|
// walk: it is unsent, and said on its own once a channel can send.
|
|
func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) {
|
|
for _, at := range []string{"end", "bound"} {
|
|
t.Run(at, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-c", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, 2*time.Minute)
|
|
down := errors.New("no session")
|
|
tg.notReady, dt.notReady = down, down
|
|
if at == "end" {
|
|
walk(t, h, "plan-c", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
} else {
|
|
runFor(h, c, 10*time.Minute+DefaultSettle) // the bound and the settle (ADR 0505 §7)
|
|
}
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said on a channel that cannot send: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || r.Pending == "" {
|
|
t.Fatalf("what the walk held is not left unsent: %+v", r)
|
|
}
|
|
tg.notReady, dt.notReady = nil, nil
|
|
runFor(h, c, 2*time.Minute)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("what the walk held was never said: %v", outcomes(h, k.Key))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 2, as novox/hq ADR 0505 §7 bounds it: a message held by walk A, about a machine walk B
|
|
// also touches, is held on at A's end while B — started after it was held — is still open, and is said once A's own
|
|
// limit (its start, bound and settle) has passed, while B still runs, with B's delivery line.
|
|
func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, novox("mesh-tools")) // its limit: 6 min and the settle
|
|
k := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, DefaultSettle+time.Minute)
|
|
if heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("told at the first walk's end while the second touches it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if got := h.heldBy(k.Key, c.now()); len(got) != 1 || got[0] != "plan-b" {
|
|
t.Fatalf("the second walk does not hold it on: %v", got)
|
|
}
|
|
runFor(h, c, 4*time.Minute) // past the first walk's limit, 8 minutes after it was held
|
|
if !heard("During the delivery of node-tools to novox.", tg, dt) || !heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("held past its first hold's limit: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 3: a router back after more than a day reads back a walk whose end never came;
|
|
// what it held is said, with the still-running line, before the walk is forgotten.
|
|
func TestAWalkReadBackADayLateSaysWhatItHeldBeforeItIsForgotten(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
c.pass(25 * time.Hour)
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
runFor(again, c, 2*time.Minute)
|
|
if !heard("is still running", dt) || !heard("- Urgent: "+headlineOf(k.Key), dt) {
|
|
t.Fatalf("what the walk held was forgotten unsaid: desk %+v, %v", dt.sends, outcomes(again, k.Key))
|
|
}
|
|
if len(again.walks) != 0 {
|
|
t.Fatalf("the walk was not forgotten: %+v", again.walks)
|
|
}
|
|
if ws, _ := st.Walks(); len(ws) != 0 {
|
|
t.Fatalf("the walk is still kept: %+v", ws)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 4 (and of mesh-controller pull request 247): a short walk's end can come before
|
|
// its start. The start of a walk already ended opens nothing — also after the router's restart — while the
|
|
// next walk under the same id, started after that end, does.
|
|
func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
started := c.now()
|
|
c.pass(20 * time.Second)
|
|
end, _, err := DecodeWalk([]byte(`{"walk":"plan-e","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
|
|
`"started_at":"` + started.Format(time.RFC3339Nano) + `","ended_at":"` + c.now().Format(time.RFC3339Nano) + `"}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Walked(end)
|
|
c.pass(time.Second)
|
|
walk(t, h, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
|
|
}
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
walk(t, again, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(again.walks) != 0 {
|
|
t.Fatalf("after a restart, a start heard after its end opened a walk: %+v", again.walks)
|
|
}
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
again.Condition(EventRaised, k)
|
|
runFor(again, c, time.Minute)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("held by a walk that had ended: %v", outcomes(again, k.Key))
|
|
}
|
|
// Retried: the same id walks again, started after the end.
|
|
walk(t, again, "plan-e", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
if len(again.walks) != 1 {
|
|
t.Fatalf("a walk started after the last end did not open: %+v", again.walks)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, non-blocking: a message whose answers became asked while a walk held it is never
|
|
// held (ADR 0308 §3); when the desk's answers are lost, it is said again in place like any other, so its
|
|
// buttons work again.
|
|
func TestARestartSaysAgainAnAskedMessageAWalkTookBeforeItsAsk(t *testing.T) {
|
|
h, _, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
stand(h, c)
|
|
walk(t, h, "plan-f", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = k.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("the ask about it was not offered: %v", outcomes(h, k.Key))
|
|
}
|
|
replaced := len(dt.replaced)
|
|
c.pass(time.Minute)
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: c.now()})
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.replaced) != replaced+1 {
|
|
t.Fatalf("its answers were not made to work again: replaced %+v, %v", dt.replaced[replaced:], outcomes(h, "(answers lost)"))
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, non-blocking: a walk's end that cannot be said as it is (here a module named by a
|
|
// file path) is said in its plain form, and what it held, still open, is said on its own, not marked told.
|
|
func TestWhatAWalksPlainEndCannotCarryIsSaidOnItsOwn(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-g", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-g", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("A delivery ended", dt) {
|
|
t.Fatalf("the plain end was not said: %+v", dt.sends)
|
|
}
|
|
if !heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("what the walk held was marked told and never said: %v", outcomes(h, k.Key))
|
|
}
|
|
}
|
|
|
|
// Issue 538 A: what a walk held is marked told only once the message carrying it was sent. When that message
|
|
// could not be sent and the router restarts before its retry, the message is gone with it: what it held is not
|
|
// marked told, it stays owed in the state, and the restarted router says it — an urgent one and a warning alike,
|
|
// once each, never left as history.
|
|
func TestAHeldMessageIsToldOnlyOnceTheWalksMessageWasSent(t *testing.T) {
|
|
for _, then := range []string{"sent", "restart urgent", "restart warning"} {
|
|
t.Run(then, func(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
sev := Urgent
|
|
if then == "restart warning" {
|
|
sev = Warning
|
|
}
|
|
walk(t, h, "plan-h", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", sev, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
down := errors.New("cannot connect")
|
|
tg.fail, dt.fail = down, down
|
|
walk(t, h, "plan-h", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said on channels that cannot send: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || r.told() {
|
|
t.Fatalf("marked told while the message carrying it was not sent: %+v", r)
|
|
}
|
|
tg.fail, dt.fail = nil, nil
|
|
if then == "sent" {
|
|
runFor(h, c, 2*time.Minute)
|
|
if !heard("Still open after the delivery of mesh-tools to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) || saidOn(tg, dt) != 2 {
|
|
t.Fatalf("not told once in the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Sent["telegram"] != "" || r.Sent["desktop"] != "" {
|
|
t.Fatalf("not marked told on both channels once sent: %+v", r)
|
|
}
|
|
runFor(h, c, 10*time.Minute)
|
|
if saidOn(tg, dt) != 2 {
|
|
t.Fatalf("said again after the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
return
|
|
}
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
again.Sync([]Condition{k}, c.now())
|
|
runFor(again, c, 5*time.Minute)
|
|
if !heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("what the lost message carried was not said after the restart: %v", outcomes(again, k.Key))
|
|
}
|
|
said := saidOn(tg, dt)
|
|
if r := again.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Pending != "" {
|
|
t.Fatalf("said, and still owed or not marked told: %+v", r)
|
|
}
|
|
runFor(again, c, 20*time.Minute) // within its reminder's bound: nothing more is due
|
|
again.Sync([]Condition{k}, c.now())
|
|
runFor(again, c, time.Minute)
|
|
if saidOn(tg, dt) != said {
|
|
t.Fatalf("said again: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Issue 538 B: at a walk's end, a later walk that touches what it held takes over only what a first hold would
|
|
// take: not a message silenced meanwhile, nor one an ask waiting for the operator is about (ADR 0308 §3).
|
|
func TestAWalksEndHandsOnOnlyWhatALaterWalkWouldHold(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
stand(h, c)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
quiet := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
quiet.Raised = c.now()
|
|
h.Condition(EventRaised, quiet)
|
|
asked := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
asked.Raised = c.now()
|
|
h.Condition(EventRaised, asked)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if heard(headlineOf(quiet.Key), tg, dt) || heard(headlineOf(asked.Key), tg, dt) {
|
|
t.Fatalf("the test's messages were not held")
|
|
}
|
|
quiet.SilencedTill, quiet.SilencedWhy, quiet.At = c.now().Add(7*24*time.Hour), "known", c.now()
|
|
h.Condition(EventChanged, quiet)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = asked.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
helds := func(key string) int {
|
|
n := 0
|
|
for _, o := range outcomes(h, key) {
|
|
if o == "held" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
before := map[string]int{quiet.Key: helds(quiet.Key), asked.Key: helds(asked.Key)}
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, DefaultSettle+time.Minute)
|
|
for _, key := range []string{quiet.Key, asked.Key} {
|
|
if got := h.heldBy(key, c.now()); len(got) != 0 {
|
|
t.Errorf("%s was handed on to %v, which a first hold would never hold", key, got)
|
|
}
|
|
if helds(key) != before[key] {
|
|
t.Errorf("the history says %s held again: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 538 C: the controller's clock ahead of the router's. A short walk's end, heard before its start, opens
|
|
// nothing when its start comes, whether the end said the walk's start or only its own moment; the next walk
|
|
// under its id, started after that end, still opens.
|
|
func TestAStartHeardAfterItsEndOpensNothingWhenTheControllersClockIsAhead(t *testing.T) {
|
|
for _, said := range []string{"both", "end only"} {
|
|
t.Run(said, func(t *testing.T) {
|
|
h, _, _, c, _ := newHolder(t)
|
|
ended := c.now().Add(10 * time.Minute) // by the controller's clock
|
|
started := ended.Add(-20 * time.Second)
|
|
body := `{"walk":"plan-k","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
|
|
`"ended_at":"` + ended.Format(time.RFC3339Nano) + `"`
|
|
if said == "both" {
|
|
body += `,"started_at":"` + started.Format(time.RFC3339Nano) + `"`
|
|
}
|
|
end, _, err := DecodeWalk([]byte(body + `}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Walked(end)
|
|
c.pass(time.Second)
|
|
walk(t, h, "plan-k", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
|
|
}
|
|
walk(t, h, "plan-k", WalkStarted, "", ended.Add(time.Minute), 600, novox("mesh-tools"))
|
|
if len(h.walks) != 1 {
|
|
t.Fatalf("the next walk under its id, started after the end, did not open: %+v", h.walks)
|
|
}
|
|
})
|
|
}
|
|
}
|