jochen
424fc5a281
mesh/merge-gate pass: builds messenger → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
A warning held on purpose and lost with the walk's message at a restart was turned into history and never told. Name the walk in the told line from its own item.
771 lines
32 KiB
Go
771 lines
32 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The tests of novox/hq ADR 0308 "How it is checked" §2–§6, with the controller's walk events given by a fake
|
|
// source: the body the controller's plan-moved carries at a walk's start and end (ADR 0308 §1).
|
|
|
|
// walkBody is a plan-moved body as the controller says a walk's start or end (mesh-controller pull request 247).
|
|
func walkBody(t *testing.T, id, phase, outcome string, at time.Time, bound int, touches ...Touch) []byte {
|
|
t.Helper()
|
|
b := map[string]any{"id": id, "state": "walking", "walk": id, "phase": phase, "touches": touches,
|
|
"updated": at.Format(time.RFC3339Nano), "budget_class": "leaf"}
|
|
if phase == WalkStarted {
|
|
b["started_at"] = at.Format(time.RFC3339Nano)
|
|
} else {
|
|
b["ended_at"] = at.Format(time.RFC3339Nano)
|
|
}
|
|
if outcome != "" {
|
|
b["outcome"] = outcome
|
|
}
|
|
if bound > 0 {
|
|
b["bound_seconds"] = bound
|
|
}
|
|
raw, err := json.Marshal(b)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// walk says a walk's start or end to the router, through the decoder the bus's events go through.
|
|
func walk(t *testing.T, h *Holder, id, phase, outcome string, at time.Time, bound int, touches ...Touch) {
|
|
t.Helper()
|
|
ev, ok, err := DecodeWalk(walkBody(t, id, phase, outcome, at, bound, touches...))
|
|
if err != nil || !ok {
|
|
t.Fatalf("the walk event was not read: %v %v", ok, err)
|
|
}
|
|
h.Walked(ev)
|
|
}
|
|
|
|
func novox(modules ...string) Touch { return Touch{Machine: "novox", Modules: modules} }
|
|
|
|
// heard says whether any message on the channels carries the words: sent, edited or said again in place.
|
|
func heard(words string, chs ...*fakeChannel) bool {
|
|
for _, ch := range chs {
|
|
for _, m := range ch.sends {
|
|
if strings.Contains(m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
for _, m := range ch.edits {
|
|
if strings.Contains(m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
for _, r := range ch.replaced {
|
|
if strings.Contains(r.m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// outcomes are the history's outcomes for a key, oldest first.
|
|
func outcomes(h *Holder, key string) []string {
|
|
var out []string
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
for _, s := range h.recent {
|
|
if s.Key == key {
|
|
out = append(out, s.Outcome)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func has(list []string, want string) bool {
|
|
for _, s := range list {
|
|
if s == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// §2: during a walk, a message about a machine or a module it touches is held, not sent, and the channel's
|
|
// history says it was held — a module's condition, a machine's, and a module's notice alike.
|
|
func TestDuringAWalkWhatItTouchesIsHeldAndTheHistorySaysSo(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("mesh-tools", "node-tools"))
|
|
mod := cond("module.node-tools.novox.unhealthy", Urgent, "the tool runner on novox does not answer")
|
|
mod.Machine, mod.Raised = "novox", c.now()
|
|
h.Condition(EventRaised, mod)
|
|
machine := cond("machine.novox.claude-code.agent-home.data-shrank", Urgent, "the agent's home shrank")
|
|
machine.Raised = c.now()
|
|
h.Condition(EventRaised, machine)
|
|
if _, err := h.Notify("tools.slow", Urgent, "tools answer slowly", "node-tools", false, "tools slow",
|
|
"Needs you: look at the tools."); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runFor(h, c, 2*time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("held messages were said: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
for _, key := range []string{mod.Key, machine.Key, "notify.tools.slow"} {
|
|
if !has(outcomes(h, key), "held") {
|
|
t.Errorf("the history does not say %s was held: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
if len(h.Open()) != 3 {
|
|
t.Fatalf("held messages are not kept open: %+v", h.Open())
|
|
}
|
|
}
|
|
|
|
// §2: a message touched by two overlapping walks is held until the second of them ends; the first's end says
|
|
// nothing of it.
|
|
func TestAMessageTouchedByTwoOverlappingWalksIsHeldUntilTheSecondEnds(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
c.pass(time.Minute)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"), Touch{Machine: "ace", Modules: []string{"node-tools"}})
|
|
k := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-tools"))
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("said at the first walk's end, while the second still holds it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if !heard("Delivered: mesh-tools on novox", tg, dt) {
|
|
t.Fatalf("the first walk's end was not told: %+v", dt.sends)
|
|
}
|
|
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("the second walk's end did not tell it: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
}
|
|
|
|
// §3: an ask, a secret given, a message about a machine no open walk touches and a walk's own failure are
|
|
// sent at once, while a walk touches the machine they are on.
|
|
func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
walk(t, h, "plan-9", WalkStarted, "", c.now(), 1200, novox("telegram", "mesh-tools"))
|
|
for _, key := range []string{
|
|
"machine.novox.telegram.telegram-token.secret-given", // a secret given, on a touched machine
|
|
"machine.novox.mounts.smb.secret-replaced", // a secret replaced
|
|
"machine.ace.silent", // a machine no walk touches
|
|
"delivery.leaf.over-budget", // the walk's own delivery
|
|
"plan.plan-9.failed", // the walk itself
|
|
} {
|
|
k := cond(key, Urgent, "it happened")
|
|
k.Raised = c.now()
|
|
if !strings.HasPrefix(key, "machine.") {
|
|
k.Machine = "novox" // a walk's own failure, on a machine it touches
|
|
}
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if !heard(headlineOf(key), tg, dt) {
|
|
t.Errorf("%s was not sent at once during the walk: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
// An ask's own message, and an ask about a held condition: both offered at once.
|
|
submit(t, h, ConditionOwner, ackAsk(c, "own"))
|
|
runFor(h, c, time.Minute)
|
|
if !has(outcomes(h, "ask."+ConditionOwner+".own"), "sent") {
|
|
t.Errorf("an ask waiting for the operator was held: %v", outcomes(h, "ask."+ConditionOwner+".own"))
|
|
}
|
|
held := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
held.Raised = c.now()
|
|
h.Condition(EventRaised, held)
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(held.Key), tg, dt) {
|
|
t.Fatalf("a condition about a touched machine was not held")
|
|
}
|
|
about := ackAsk(c, "about")
|
|
about.About = held.Key
|
|
submit(t, h, ConditionOwner, about)
|
|
runFor(h, c, time.Minute)
|
|
if !heard(headlineOf(held.Key), dt) {
|
|
t.Fatalf("the ask about a held condition was held: %v", outcomes(h, held.Key))
|
|
}
|
|
}
|
|
|
|
// §4: a walk's end is one message — its outcome, what it reached and how long it took — then each held message
|
|
// still open, one line each; the one that cleared is only counted, and nothing more is said of them after.
|
|
func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-3", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
still := cond("machine.novox.tools-silent", Warning, "the tools on novox are silent")
|
|
still.Raised = c.now()
|
|
h.Condition(EventRaised, still)
|
|
gone := cond("module.mesh-tools.novox.unhealthy", Warning, "mesh-tools is unhealthy on novox")
|
|
gone.Raised = c.now()
|
|
h.Condition(EventRaised, gone)
|
|
c.pass(time.Second)
|
|
other := cond("machine.novox.core-behind", Urgent, "novox runs an older core")
|
|
other.Raised = c.now()
|
|
h.Condition(EventRaised, other)
|
|
runFor(h, c, 2*time.Minute)
|
|
h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()})
|
|
runFor(h, c, 5*time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said during the walk: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
walk(t, h, "plan-3", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
// Urgent among what it holds: on every kind, one message each.
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 {
|
|
t.Fatalf("the end was not one message per kind: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
m := dt.sends[0]
|
|
if m.Title != "Delivery failed: mesh-tools on novox" || !strings.Contains(m.Body, "It took 7 min.") {
|
|
t.Fatalf("the end does not say its outcome, what it reached and how long: %q", m.Text())
|
|
}
|
|
want := "Still open after it:\n- " + headlineOf(still.Key) + "\n- Urgent: " + headlineOf(other.Key) +
|
|
"\n1 problem it raised resolved during it."
|
|
if !strings.Contains(m.Body, want) || strings.Contains(m.Text(), headlineOf(gone.Key)) {
|
|
t.Fatalf("the end message %q does not say %q", m.Body, want)
|
|
}
|
|
keyless(t, m)
|
|
runFor(h, c, 20*time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 || len(dt.replaced)+len(tg.replaced) != 0 {
|
|
t.Fatalf("what the end told was said again: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
}
|
|
|
|
// §4: a walk that delivered with nothing held and nothing open is one line, and nothing at all on a kind set
|
|
// to say only what needs the operator.
|
|
func TestADeliveredWalkWithNothingHeldIsOneLineOrNothing(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-4", WalkStarted, "", c.now(), 600, novox("mesh-tools"), Touch{Machine: "ace", Modules: []string{"mesh-tools"}})
|
|
runFor(h, c, 3*time.Minute)
|
|
walk(t, h, "plan-4", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
if m := dt.sends[0]; m.Title != "Delivered: mesh-tools on ace, novox, in 3 min" || m.Body != "" {
|
|
t.Fatalf("not one line: %q", m.Text())
|
|
}
|
|
h.NeedsOnly = func(kind string) bool { return kind == "desktop" }
|
|
walk(t, h, "plan-5", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
runFor(h, c, 3*time.Minute)
|
|
walk(t, h, "plan-5", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("told on a kind that says only what needs the operator: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
if !has(outcomes(h, "walk.plan-5"), "quiet: this kind says only what needs the operator") {
|
|
t.Fatalf("the history does not say why nothing was told: %v", outcomes(h, "walk.plan-5"))
|
|
}
|
|
}
|
|
|
|
// §5: the bound is the controller's (bound_seconds: twice the largest budget of the walk's deliveries; none
|
|
// given, 15 minutes). At it, what is held and still open is sent with the still-running line, and what
|
|
// cleared is counted; before it, nothing; after it, the walk holds nothing more.
|
|
func TestAHoldEndsAtTheWalksBound(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
given int
|
|
bound time.Duration
|
|
}{
|
|
{"core: twice ten minutes", 1200, 20 * time.Minute},
|
|
{"leaf at five minutes", 600, 10 * time.Minute},
|
|
{"leaf at three minutes", 360, 6 * time.Minute},
|
|
{"no budget", 0, 15 * time.Minute},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
started := c.now()
|
|
walk(t, h, "plan-7", WalkStarted, "", started, tc.given, novox("node-tools"))
|
|
open := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
open.Raised = c.now()
|
|
h.Condition(EventRaised, open)
|
|
gone := cond("module.node-tools.novox.unhealthy", Urgent, "node-tools is unhealthy")
|
|
gone.Raised = c.now()
|
|
h.Condition(EventRaised, gone)
|
|
c.pass(30 * time.Second)
|
|
h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()})
|
|
runFor(h, c, tc.bound-time.Minute-30*time.Second)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said before the bound: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
runFor(h, c, 2*time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 {
|
|
t.Fatalf("at the bound: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
m := dt.sends[0]
|
|
since := started.In(h.zone()).Format("15:04")
|
|
for _, want := range []string{"is still running, since " + since + ".", "- Urgent: " + headlineOf(open.Key),
|
|
"1 problem it raised resolved during it."} {
|
|
if !strings.Contains(m.Body, want) {
|
|
t.Fatalf("the bound's message %q does not say %q", m.Text(), want)
|
|
}
|
|
}
|
|
// Past its bound the walk holds nothing: a new message about what it touches goes at once.
|
|
late := cond("machine.novox.disk", Urgent, "a disk fails on novox")
|
|
late.Raised = c.now()
|
|
h.Condition(EventRaised, late)
|
|
runFor(h, c, time.Minute)
|
|
if !heard(headlineOf(late.Key), dt) {
|
|
t.Fatalf("held past the bound: %v", outcomes(h, late.Key))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §2, kept: what a walk holds survives the router's restart, and is told at the walk's end.
|
|
func TestHeldMessagesSurviveARouterRestart(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
walk(t, h, "plan-8", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
c.pass(2 * time.Minute)
|
|
walk(t, again, "plan-8", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(again, c, time.Minute)
|
|
if !heard("- Urgent: "+headlineOf(k.Key), dt) || saidOn(tg, dt) != 2 {
|
|
t.Fatalf("what was held before the restart was not told at the walk's end: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
}
|
|
|
|
// §6, issue 531: the desk's notifiers on two machines start again a second apart, with four silenced messages
|
|
// shown there — one of them withheld for its content, a file path — and a fifth open, asked there, and
|
|
// withheld. Nothing is said again: no silenced, no withheld message; the pass runs once; and a router that
|
|
// starts again reads it back from its state and makes it no more.
|
|
func TestARestartSaysNothingItHasSaid(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
c.pass(2 * time.Minute)
|
|
keys := []string{"machine.novox.telegram.telegram-token.secret-given", "machine.novox.claude-code.agent-home.data-shrank",
|
|
"machine.shanks.mounts.smb-password-recalbox.secret-given", "machine.ace.nzbgeek.api-key.secret-given"}
|
|
var silenced []Condition
|
|
for i, key := range keys {
|
|
k := cond(key, Urgent, "it happened")
|
|
if i == 1 {
|
|
k.Explanation = "Needs you: look at /var/lib/agent-home, which shrank."
|
|
}
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
silenced = append(silenced, k)
|
|
}
|
|
withheldOpen := cond("machine.novox.units", Urgent, "2 failed services")
|
|
withheldOpen.Explanation = "Needs you: read /etc/systemd/system/x.service."
|
|
withheldOpen.Raised = c.now()
|
|
h.Condition(EventRaised, withheldOpen)
|
|
runFor(h, c, time.Minute)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = withheldOpen.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
for _, k := range silenced {
|
|
k.SilencedTill, k.SilencedWhy, k.At = c.now().Add(7*24*time.Hour), "known", c.now()
|
|
h.Condition(EventChanged, k)
|
|
}
|
|
runFor(h, c, time.Minute)
|
|
for _, key := range []string{silenced[1].Key, withheldOpen.Key} {
|
|
if r := h.open[key]; r == nil || r.Refused == "" {
|
|
t.Fatalf("the test's %s was not withheld for its content: %+v", key, r)
|
|
}
|
|
}
|
|
sends, edits, replaced, tgSaid := len(dt.sends), len(dt.edits), len(dt.replaced), saidOn(tg)
|
|
dt.edits = map[string]Message{}
|
|
passes := func(h *Holder) int { return len(outcomes(h, "(answers lost)")) }
|
|
// The notifiers on two machines started again a second apart: two standings, two times.
|
|
c.pass(time.Minute)
|
|
first := c.now()
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: first})
|
|
c.pass(time.Second)
|
|
h.Flush() // the runtime's tick, between the two standings
|
|
second := c.now()
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second})
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid {
|
|
t.Fatalf("the restart said again: sends %+v, edits %+v (was %d), replaced %+v", dt.sends[sends:], dt.edits, edits,
|
|
dt.replaced[replaced:])
|
|
}
|
|
if passes(h) != 1 {
|
|
t.Fatalf("the pass ran %d times for one restart", passes(h))
|
|
}
|
|
// The router starts again; the desk repeats its standing.
|
|
again := &Holder{Away: tg, Desk: dt, Store: h.Store, Asks: h.Asks, Bench: h.Bench, Publish: h.Publish, Now: c.now,
|
|
Logf: t.Logf, Zone: h.Zone, RootFree: h.RootFree, Node: h.Node}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
again.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second})
|
|
runFor(again, c, 2*time.Minute)
|
|
if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid || passes(again) != 1 {
|
|
t.Fatalf("the router's restart made the pass again: %d passes, sends %+v, edits %+v", passes(again),
|
|
dt.sends[sends:], dt.edits)
|
|
}
|
|
}
|
|
|
|
// The controller's walk events are read as ADR 0308 §1 says them; any other plan-moved is not a walk's.
|
|
func TestTheWalkEventsAreReadAsTheControllerSaysThem(t *testing.T) {
|
|
at := time.Date(2026, 10, 12, 11, 12, 0, 0, time.UTC)
|
|
ev, ok, err := DecodeWalk(walkBody(t, "plan-1", WalkStarted, "", at, 1200, novox("mesh-tools", "node-tools")))
|
|
if err != nil || !ok || ev.Walk != "plan-1" || ev.BoundSeconds != 1200 || ev.Class != "leaf" || !ev.At.Equal(at) ||
|
|
len(ev.Touches) != 1 || ev.Touches[0].Machine != "novox" || len(ev.Touches[0].Modules) != 2 {
|
|
t.Fatalf("%+v %v %v", ev, ok, err)
|
|
}
|
|
// As mesh-controller's walk events (its pull request 247) say an end: ended_at, and carried beside modules.
|
|
end, ok, err := DecodeWalk([]byte(`{"id":"plan-1","walk":"plan-1","phase":"ended","outcome":"stopped",` +
|
|
`"touches":[{"machine":"novox","modules":["node-tools","messenger"],"carried":["messenger"]}],` +
|
|
`"started_at":"2026-10-12T11:12:00Z","ended_at":"2026-10-12T11:19:00Z","updated":"2026-10-12T11:19:01Z"}`))
|
|
if err != nil || !ok || end.Outcome != WalkStopped || !end.At.Equal(at.Add(7*time.Minute)) || len(end.Touches[0].Modules) != 2 {
|
|
t.Fatalf("%+v %v %v", end, ok, err)
|
|
}
|
|
if _, ok, err := DecodeWalk([]byte(`{"id":"plan-1","state":"walking","tier":1}`)); ok || err != nil {
|
|
t.Fatalf("a move inside a walk read as its start or end: %v %v", ok, err)
|
|
}
|
|
if _, _, err := DecodeWalk([]byte(`{"walk":"plan-1","phase":"ended"}`)); err == nil {
|
|
t.Fatalf("an end without its outcome was read")
|
|
}
|
|
w := Walk{}
|
|
if w.bound() != DefaultWalkBound {
|
|
t.Fatalf("a walk without a bound is bounded at %s", w.bound())
|
|
}
|
|
// A start older than its bound is history: nothing is held.
|
|
h, _, _, c, _ := newHolder(t)
|
|
walk(t, h, "plan-old", WalkStarted, "", c.now().Add(-time.Hour), 1200, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("an old start opened a walk: %+v", h.walks)
|
|
}
|
|
|
|
}
|
|
|
|
// Review of pull request 230, 1: no channel can send at a walk's end or bound. What it held is not lost with the
|
|
// walk: it is unsent, and said on its own once a channel can send.
|
|
func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) {
|
|
for _, at := range []string{"end", "bound"} {
|
|
t.Run(at, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-c", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, 2*time.Minute)
|
|
down := errors.New("no session")
|
|
tg.notReady, dt.notReady = down, down
|
|
if at == "end" {
|
|
walk(t, h, "plan-c", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
} else {
|
|
runFor(h, c, 10*time.Minute)
|
|
}
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said on a channel that cannot send: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || r.Pending == "" {
|
|
t.Fatalf("what the walk held is not left unsent: %+v", r)
|
|
}
|
|
tg.notReady, dt.notReady = nil, nil
|
|
runFor(h, c, 2*time.Minute)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("what the walk held was never said: %v", outcomes(h, k.Key))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 2: a message held by walk A, about a machine walk B also touches, is held on at
|
|
// A's end while B — started after it was held — is still open, and told at B's end.
|
|
func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("told at the first walk's end while the second touches it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if got := h.heldBy(k.Key, c.now()); len(got) != 1 || got[0] != "plan-b" {
|
|
t.Fatalf("the second walk does not hold it on: %v", got)
|
|
}
|
|
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("the second walk's end did not tell it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 3: a router back after more than a day reads back a walk whose end never came;
|
|
// what it held is said, with the still-running line, before the walk is forgotten.
|
|
func TestAWalkReadBackADayLateSaysWhatItHeldBeforeItIsForgotten(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
c.pass(25 * time.Hour)
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
runFor(again, c, 2*time.Minute)
|
|
if !heard("is still running", dt) || !heard("- Urgent: "+headlineOf(k.Key), dt) {
|
|
t.Fatalf("what the walk held was forgotten unsaid: desk %+v, %v", dt.sends, outcomes(again, k.Key))
|
|
}
|
|
if len(again.walks) != 0 {
|
|
t.Fatalf("the walk was not forgotten: %+v", again.walks)
|
|
}
|
|
if ws, _ := st.Walks(); len(ws) != 0 {
|
|
t.Fatalf("the walk is still kept: %+v", ws)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 4 (and of mesh-controller pull request 247): a short walk's end can come before
|
|
// its start. The start of a walk already ended opens nothing — also after the router's restart — while the
|
|
// next walk under the same id, started after that end, does.
|
|
func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
started := c.now()
|
|
c.pass(20 * time.Second)
|
|
end, _, err := DecodeWalk([]byte(`{"walk":"plan-e","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
|
|
`"started_at":"` + started.Format(time.RFC3339Nano) + `","ended_at":"` + c.now().Format(time.RFC3339Nano) + `"}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Walked(end)
|
|
c.pass(time.Second)
|
|
walk(t, h, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
|
|
}
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
walk(t, again, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(again.walks) != 0 {
|
|
t.Fatalf("after a restart, a start heard after its end opened a walk: %+v", again.walks)
|
|
}
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
again.Condition(EventRaised, k)
|
|
runFor(again, c, time.Minute)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("held by a walk that had ended: %v", outcomes(again, k.Key))
|
|
}
|
|
// Retried: the same id walks again, started after the end.
|
|
walk(t, again, "plan-e", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
if len(again.walks) != 1 {
|
|
t.Fatalf("a walk started after the last end did not open: %+v", again.walks)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, non-blocking: a message whose answers became asked while a walk held it is never
|
|
// held (ADR 0308 §3); when the desk's answers are lost, it is said again in place like any other, so its
|
|
// buttons work again.
|
|
func TestARestartSaysAgainAnAskedMessageAWalkTookBeforeItsAsk(t *testing.T) {
|
|
h, _, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
stand(h, c)
|
|
walk(t, h, "plan-f", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = k.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("the ask about it was not offered: %v", outcomes(h, k.Key))
|
|
}
|
|
replaced := len(dt.replaced)
|
|
c.pass(time.Minute)
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: c.now()})
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.replaced) != replaced+1 {
|
|
t.Fatalf("its answers were not made to work again: replaced %+v, %v", dt.replaced[replaced:], outcomes(h, "(answers lost)"))
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, non-blocking: a walk's end that cannot be said as it is (here a module named by a
|
|
// file path) is said in its plain form, and what it held, still open, is said on its own, not marked told.
|
|
func TestWhatAWalksPlainEndCannotCarryIsSaidOnItsOwn(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-g", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-g", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("A delivery ended", dt) {
|
|
t.Fatalf("the plain end was not said: %+v", dt.sends)
|
|
}
|
|
if !heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("what the walk held was marked told and never said: %v", outcomes(h, k.Key))
|
|
}
|
|
}
|
|
|
|
// Issue 538 A: what a walk held is marked told only once the message carrying it was sent. When that message
|
|
// could not be sent and the router restarts before its retry, the message is gone with it: what it held is not
|
|
// marked told, it stays owed in the state, and the restarted router says it — an urgent one and a warning alike,
|
|
// once each, never left as history.
|
|
func TestAHeldMessageIsToldOnlyOnceTheWalksMessageWasSent(t *testing.T) {
|
|
for _, then := range []string{"sent", "restart urgent", "restart warning"} {
|
|
t.Run(then, func(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
sev := Urgent
|
|
if then == "restart warning" {
|
|
sev = Warning
|
|
}
|
|
walk(t, h, "plan-h", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", sev, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
down := errors.New("cannot connect")
|
|
tg.fail, dt.fail = down, down
|
|
walk(t, h, "plan-h", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said on channels that cannot send: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || r.told() {
|
|
t.Fatalf("marked told while the message carrying it was not sent: %+v", r)
|
|
}
|
|
tg.fail, dt.fail = nil, nil
|
|
if then == "sent" {
|
|
runFor(h, c, 2*time.Minute)
|
|
if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) || saidOn(tg, dt) != 2 {
|
|
t.Fatalf("not told once in the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Sent["telegram"] != "" || r.Sent["desktop"] != "" {
|
|
t.Fatalf("not marked told on both channels once sent: %+v", r)
|
|
}
|
|
runFor(h, c, 10*time.Minute)
|
|
if saidOn(tg, dt) != 2 {
|
|
t.Fatalf("said again after the walk's message: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
return
|
|
}
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
again.Sync([]Condition{k}, c.now())
|
|
runFor(again, c, 5*time.Minute)
|
|
if !heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("what the lost message carried was not said after the restart: %v", outcomes(again, k.Key))
|
|
}
|
|
said := saidOn(tg, dt)
|
|
if r := again.open[k.Key]; r == nil || !r.told() || r.Owed != "" || r.Pending != "" {
|
|
t.Fatalf("said, and still owed or not marked told: %+v", r)
|
|
}
|
|
runFor(again, c, 20*time.Minute) // within its reminder's bound: nothing more is due
|
|
again.Sync([]Condition{k}, c.now())
|
|
runFor(again, c, time.Minute)
|
|
if saidOn(tg, dt) != said {
|
|
t.Fatalf("said again: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Issue 538 B: at a walk's end, a later walk that touches what it held takes over only what a first hold would
|
|
// take: not a message silenced meanwhile, nor one an ask waiting for the operator is about (ADR 0308 §3).
|
|
func TestAWalksEndHandsOnOnlyWhatALaterWalkWouldHold(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
stand(h, c)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
quiet := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
quiet.Raised = c.now()
|
|
h.Condition(EventRaised, quiet)
|
|
asked := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
asked.Raised = c.now()
|
|
h.Condition(EventRaised, asked)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if heard(headlineOf(quiet.Key), tg, dt) || heard(headlineOf(asked.Key), tg, dt) {
|
|
t.Fatalf("the test's messages were not held")
|
|
}
|
|
quiet.SilencedTill, quiet.SilencedWhy, quiet.At = c.now().Add(7*24*time.Hour), "known", c.now()
|
|
h.Condition(EventChanged, quiet)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = asked.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
helds := func(key string) int {
|
|
n := 0
|
|
for _, o := range outcomes(h, key) {
|
|
if o == "held" {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
before := map[string]int{quiet.Key: helds(quiet.Key), asked.Key: helds(asked.Key)}
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
for _, key := range []string{quiet.Key, asked.Key} {
|
|
if got := h.heldBy(key, c.now()); len(got) != 0 {
|
|
t.Errorf("%s was handed on to %v, which a first hold would never hold", key, got)
|
|
}
|
|
if helds(key) != before[key] {
|
|
t.Errorf("the history says %s held again: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue 538 C: the controller's clock ahead of the router's. A short walk's end, heard before its start, opens
|
|
// nothing when its start comes, whether the end said the walk's start or only its own moment; the next walk
|
|
// under its id, started after that end, still opens.
|
|
func TestAStartHeardAfterItsEndOpensNothingWhenTheControllersClockIsAhead(t *testing.T) {
|
|
for _, said := range []string{"both", "end only"} {
|
|
t.Run(said, func(t *testing.T) {
|
|
h, _, _, c, _ := newHolder(t)
|
|
ended := c.now().Add(10 * time.Minute) // by the controller's clock
|
|
started := ended.Add(-20 * time.Second)
|
|
body := `{"walk":"plan-k","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
|
|
`"ended_at":"` + ended.Format(time.RFC3339Nano) + `"`
|
|
if said == "both" {
|
|
body += `,"started_at":"` + started.Format(time.RFC3339Nano) + `"`
|
|
}
|
|
end, _, err := DecodeWalk([]byte(body + `}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Walked(end)
|
|
c.pass(time.Second)
|
|
walk(t, h, "plan-k", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
|
|
}
|
|
walk(t, h, "plan-k", WalkStarted, "", ended.Add(time.Minute), 600, novox("mesh-tools"))
|
|
if len(h.walks) != 1 {
|
|
t.Fatalf("the next walk under its id, started after the end, did not open: %+v", h.walks)
|
|
}
|
|
})
|
|
}
|
|
}
|