jochen
a92c08fd5b
mesh/merge-gate pass: builds messenger → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
- No channel ready at a walk's end or bound: what it held is left unsent and tried again every minute, as any other message, instead of lost with the walk. - At a walk's end or bound, a message another open walk touches is held on by that walk, even one started after it was held (hq ADR 0308 §2). - A walk read back past its forget is released at its bound first, then forgotten. - A walk's end is kept a day: a start heard after it (same started_at, or not later than the end) opens nothing, also after the router restarts; a later start under the same id still opens. - A walk's plain end, when its words cannot be said, leaves what it held to be said on its own instead of marking it told. - The restart pass no longer skips a held message: one with an open ask is never held (ADR 0308 §3), so its answers are made to work again.
623 lines
26 KiB
Go
623 lines
26 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The tests of novox/hq ADR 0308 "How it is checked" §2–§6, with the controller's walk events given by a fake
|
|
// source: the body the controller's plan-moved carries at a walk's start and end (ADR 0308 §1).
|
|
|
|
// walkBody is a plan-moved body as the controller says a walk's start or end (mesh-controller pull request 247).
|
|
func walkBody(t *testing.T, id, phase, outcome string, at time.Time, bound int, touches ...Touch) []byte {
|
|
t.Helper()
|
|
b := map[string]any{"id": id, "state": "walking", "walk": id, "phase": phase, "touches": touches,
|
|
"updated": at.Format(time.RFC3339Nano), "budget_class": "leaf"}
|
|
if phase == WalkStarted {
|
|
b["started_at"] = at.Format(time.RFC3339Nano)
|
|
} else {
|
|
b["ended_at"] = at.Format(time.RFC3339Nano)
|
|
}
|
|
if outcome != "" {
|
|
b["outcome"] = outcome
|
|
}
|
|
if bound > 0 {
|
|
b["bound_seconds"] = bound
|
|
}
|
|
raw, err := json.Marshal(b)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// walk says a walk's start or end to the router, through the decoder the bus's events go through.
|
|
func walk(t *testing.T, h *Holder, id, phase, outcome string, at time.Time, bound int, touches ...Touch) {
|
|
t.Helper()
|
|
ev, ok, err := DecodeWalk(walkBody(t, id, phase, outcome, at, bound, touches...))
|
|
if err != nil || !ok {
|
|
t.Fatalf("the walk event was not read: %v %v", ok, err)
|
|
}
|
|
h.Walked(ev)
|
|
}
|
|
|
|
func novox(modules ...string) Touch { return Touch{Machine: "novox", Modules: modules} }
|
|
|
|
// heard says whether any message on the channels carries the words: sent, edited or said again in place.
|
|
func heard(words string, chs ...*fakeChannel) bool {
|
|
for _, ch := range chs {
|
|
for _, m := range ch.sends {
|
|
if strings.Contains(m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
for _, m := range ch.edits {
|
|
if strings.Contains(m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
for _, r := range ch.replaced {
|
|
if strings.Contains(r.m.Text(), words) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// outcomes are the history's outcomes for a key, oldest first.
|
|
func outcomes(h *Holder, key string) []string {
|
|
var out []string
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
for _, s := range h.recent {
|
|
if s.Key == key {
|
|
out = append(out, s.Outcome)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func has(list []string, want string) bool {
|
|
for _, s := range list {
|
|
if s == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// §2: during a walk, a message about a machine or a module it touches is held, not sent, and the channel's
|
|
// history says it was held — a module's condition, a machine's, and a module's notice alike.
|
|
func TestDuringAWalkWhatItTouchesIsHeldAndTheHistorySaysSo(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("mesh-tools", "node-tools"))
|
|
mod := cond("module.node-tools.novox.unhealthy", Urgent, "the tool runner on novox does not answer")
|
|
mod.Machine, mod.Raised = "novox", c.now()
|
|
h.Condition(EventRaised, mod)
|
|
machine := cond("machine.novox.claude-code.agent-home.data-shrank", Urgent, "the agent's home shrank")
|
|
machine.Raised = c.now()
|
|
h.Condition(EventRaised, machine)
|
|
if _, err := h.Notify("tools.slow", Urgent, "tools answer slowly", "node-tools", false, "tools slow",
|
|
"Needs you: look at the tools."); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runFor(h, c, 2*time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("held messages were said: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
for _, key := range []string{mod.Key, machine.Key, "notify.tools.slow"} {
|
|
if !has(outcomes(h, key), "held") {
|
|
t.Errorf("the history does not say %s was held: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
if len(h.Open()) != 3 {
|
|
t.Fatalf("held messages are not kept open: %+v", h.Open())
|
|
}
|
|
}
|
|
|
|
// §2: a message touched by two overlapping walks is held until the second of them ends; the first's end says
|
|
// nothing of it.
|
|
func TestAMessageTouchedByTwoOverlappingWalksIsHeldUntilTheSecondEnds(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
c.pass(time.Minute)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"), Touch{Machine: "ace", Modules: []string{"node-tools"}})
|
|
k := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-tools"))
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("said at the first walk's end, while the second still holds it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if !heard("Delivered: mesh-tools on novox", tg, dt) {
|
|
t.Fatalf("the first walk's end was not told: %+v", dt.sends)
|
|
}
|
|
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("the second walk's end did not tell it: telegram %+v, desk %+v", tg.sends, dt.sends)
|
|
}
|
|
}
|
|
|
|
// §3: an ask, a secret given, a message about a machine no open walk touches and a walk's own failure are
|
|
// sent at once, while a walk touches the machine they are on.
|
|
func TestWhatAWalkNeverHoldsIsSentAtOnce(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
walk(t, h, "plan-9", WalkStarted, "", c.now(), 1200, novox("telegram", "mesh-tools"))
|
|
for _, key := range []string{
|
|
"machine.novox.telegram.telegram-token.secret-given", // a secret given, on a touched machine
|
|
"machine.novox.mounts.smb.secret-replaced", // a secret replaced
|
|
"machine.ace.silent", // a machine no walk touches
|
|
"delivery.leaf.over-budget", // the walk's own delivery
|
|
"plan.plan-9.failed", // the walk itself
|
|
} {
|
|
k := cond(key, Urgent, "it happened")
|
|
k.Raised = c.now()
|
|
if !strings.HasPrefix(key, "machine.") {
|
|
k.Machine = "novox" // a walk's own failure, on a machine it touches
|
|
}
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if !heard(headlineOf(key), tg, dt) {
|
|
t.Errorf("%s was not sent at once during the walk: %v", key, outcomes(h, key))
|
|
}
|
|
}
|
|
// An ask's own message, and an ask about a held condition: both offered at once.
|
|
submit(t, h, ConditionOwner, ackAsk(c, "own"))
|
|
runFor(h, c, time.Minute)
|
|
if !has(outcomes(h, "ask."+ConditionOwner+".own"), "sent") {
|
|
t.Errorf("an ask waiting for the operator was held: %v", outcomes(h, "ask."+ConditionOwner+".own"))
|
|
}
|
|
held := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
held.Raised = c.now()
|
|
h.Condition(EventRaised, held)
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(held.Key), tg, dt) {
|
|
t.Fatalf("a condition about a touched machine was not held")
|
|
}
|
|
about := ackAsk(c, "about")
|
|
about.About = held.Key
|
|
submit(t, h, ConditionOwner, about)
|
|
runFor(h, c, time.Minute)
|
|
if !heard(headlineOf(held.Key), dt) {
|
|
t.Fatalf("the ask about a held condition was held: %v", outcomes(h, held.Key))
|
|
}
|
|
}
|
|
|
|
// §4: a walk's end is one message — its outcome, what it reached and how long it took — then each held message
|
|
// still open, one line each; the one that cleared is only counted, and nothing more is said of them after.
|
|
func TestAWalksEndIsToldOnceWithWhatIsStillOpen(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-3", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
still := cond("machine.novox.tools-silent", Warning, "the tools on novox are silent")
|
|
still.Raised = c.now()
|
|
h.Condition(EventRaised, still)
|
|
gone := cond("module.mesh-tools.novox.unhealthy", Warning, "mesh-tools is unhealthy on novox")
|
|
gone.Raised = c.now()
|
|
h.Condition(EventRaised, gone)
|
|
c.pass(time.Second)
|
|
other := cond("machine.novox.core-behind", Urgent, "novox runs an older core")
|
|
other.Raised = c.now()
|
|
h.Condition(EventRaised, other)
|
|
runFor(h, c, 2*time.Minute)
|
|
h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()})
|
|
runFor(h, c, 5*time.Minute)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said during the walk: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
walk(t, h, "plan-3", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
// Urgent among what it holds: on every kind, one message each.
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 {
|
|
t.Fatalf("the end was not one message per kind: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
m := dt.sends[0]
|
|
if m.Title != "Delivery failed: mesh-tools on novox" || !strings.Contains(m.Body, "It took 7 min.") {
|
|
t.Fatalf("the end does not say its outcome, what it reached and how long: %q", m.Text())
|
|
}
|
|
want := "Still open after it:\n- " + headlineOf(still.Key) + "\n- Urgent: " + headlineOf(other.Key) +
|
|
"\n1 problem it raised resolved during it."
|
|
if !strings.Contains(m.Body, want) || strings.Contains(m.Text(), headlineOf(gone.Key)) {
|
|
t.Fatalf("the end message %q does not say %q", m.Body, want)
|
|
}
|
|
keyless(t, m)
|
|
runFor(h, c, 20*time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 || len(dt.replaced)+len(tg.replaced) != 0 {
|
|
t.Fatalf("what the end told was said again: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
}
|
|
|
|
// §4: a walk that delivered with nothing held and nothing open is one line, and nothing at all on a kind set
|
|
// to say only what needs the operator.
|
|
func TestADeliveredWalkWithNothingHeldIsOneLineOrNothing(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-4", WalkStarted, "", c.now(), 600, novox("mesh-tools"), Touch{Machine: "ace", Modules: []string{"mesh-tools"}})
|
|
runFor(h, c, 3*time.Minute)
|
|
walk(t, h, "plan-4", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
if m := dt.sends[0]; m.Title != "Delivered: mesh-tools on ace, novox, in 3 min" || m.Body != "" {
|
|
t.Fatalf("not one line: %q", m.Text())
|
|
}
|
|
h.NeedsOnly = func(kind string) bool { return kind == "desktop" }
|
|
walk(t, h, "plan-5", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
runFor(h, c, 3*time.Minute)
|
|
walk(t, h, "plan-5", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 0 {
|
|
t.Fatalf("told on a kind that says only what needs the operator: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
if !has(outcomes(h, "walk.plan-5"), "quiet: this kind says only what needs the operator") {
|
|
t.Fatalf("the history does not say why nothing was told: %v", outcomes(h, "walk.plan-5"))
|
|
}
|
|
}
|
|
|
|
// §5: the bound is the controller's (bound_seconds: twice the largest budget of the walk's deliveries; none
|
|
// given, 15 minutes). At it, what is held and still open is sent with the still-running line, and what
|
|
// cleared is counted; before it, nothing; after it, the walk holds nothing more.
|
|
func TestAHoldEndsAtTheWalksBound(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
given int
|
|
bound time.Duration
|
|
}{
|
|
{"core: twice ten minutes", 1200, 20 * time.Minute},
|
|
{"leaf at five minutes", 600, 10 * time.Minute},
|
|
{"leaf at three minutes", 360, 6 * time.Minute},
|
|
{"no budget", 0, 15 * time.Minute},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
started := c.now()
|
|
walk(t, h, "plan-7", WalkStarted, "", started, tc.given, novox("node-tools"))
|
|
open := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
open.Raised = c.now()
|
|
h.Condition(EventRaised, open)
|
|
gone := cond("module.node-tools.novox.unhealthy", Urgent, "node-tools is unhealthy")
|
|
gone.Raised = c.now()
|
|
h.Condition(EventRaised, gone)
|
|
c.pass(30 * time.Second)
|
|
h.Condition(EventCleared, Condition{Key: gone.Key, At: c.now()})
|
|
runFor(h, c, tc.bound-time.Minute-30*time.Second)
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said before the bound: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
runFor(h, c, 2*time.Minute)
|
|
if len(dt.sends) != 1 || len(tg.sends) != 1 {
|
|
t.Fatalf("at the bound: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
m := dt.sends[0]
|
|
since := started.In(h.zone()).Format("15:04")
|
|
for _, want := range []string{"is still running, since " + since + ".", "- Urgent: " + headlineOf(open.Key),
|
|
"1 problem it raised resolved during it."} {
|
|
if !strings.Contains(m.Body, want) {
|
|
t.Fatalf("the bound's message %q does not say %q", m.Text(), want)
|
|
}
|
|
}
|
|
// Past its bound the walk holds nothing: a new message about what it touches goes at once.
|
|
late := cond("machine.novox.disk", Urgent, "a disk fails on novox")
|
|
late.Raised = c.now()
|
|
h.Condition(EventRaised, late)
|
|
runFor(h, c, time.Minute)
|
|
if !heard(headlineOf(late.Key), dt) {
|
|
t.Fatalf("held past the bound: %v", outcomes(h, late.Key))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// §2, kept: what a walk holds survives the router's restart, and is told at the walk's end.
|
|
func TestHeldMessagesSurviveARouterRestart(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
walk(t, h, "plan-8", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
c.pass(2 * time.Minute)
|
|
walk(t, again, "plan-8", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(again, c, time.Minute)
|
|
if !heard("- Urgent: "+headlineOf(k.Key), dt) || saidOn(tg, dt) != 2 {
|
|
t.Fatalf("what was held before the restart was not told at the walk's end: desk %+v, telegram %+v", dt.sends, tg.sends)
|
|
}
|
|
}
|
|
|
|
// §6, issue 531: the desk's notifiers on two machines start again a second apart, with four silenced messages
|
|
// shown there — one of them withheld for its content, a file path — and a fifth open, asked there, and
|
|
// withheld. Nothing is said again: no silenced, no withheld message; the pass runs once; and a router that
|
|
// starts again reads it back from its state and makes it no more.
|
|
func TestARestartSaysNothingItHasSaid(t *testing.T) {
|
|
h, tg, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
c.pass(2 * time.Minute)
|
|
keys := []string{"machine.novox.telegram.telegram-token.secret-given", "machine.novox.claude-code.agent-home.data-shrank",
|
|
"machine.shanks.mounts.smb-password-recalbox.secret-given", "machine.ace.nzbgeek.api-key.secret-given"}
|
|
var silenced []Condition
|
|
for i, key := range keys {
|
|
k := cond(key, Urgent, "it happened")
|
|
if i == 1 {
|
|
k.Explanation = "Needs you: look at /var/lib/agent-home, which shrank."
|
|
}
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
silenced = append(silenced, k)
|
|
}
|
|
withheldOpen := cond("machine.novox.units", Urgent, "2 failed services")
|
|
withheldOpen.Explanation = "Needs you: read /etc/systemd/system/x.service."
|
|
withheldOpen.Raised = c.now()
|
|
h.Condition(EventRaised, withheldOpen)
|
|
runFor(h, c, time.Minute)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = withheldOpen.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
for _, k := range silenced {
|
|
k.SilencedTill, k.SilencedWhy, k.At = c.now().Add(7*24*time.Hour), "known", c.now()
|
|
h.Condition(EventChanged, k)
|
|
}
|
|
runFor(h, c, time.Minute)
|
|
for _, key := range []string{silenced[1].Key, withheldOpen.Key} {
|
|
if r := h.open[key]; r == nil || r.Refused == "" {
|
|
t.Fatalf("the test's %s was not withheld for its content: %+v", key, r)
|
|
}
|
|
}
|
|
sends, edits, replaced, tgSaid := len(dt.sends), len(dt.edits), len(dt.replaced), saidOn(tg)
|
|
dt.edits = map[string]Message{}
|
|
passes := func(h *Holder) int { return len(outcomes(h, "(answers lost)")) }
|
|
// The notifiers on two machines started again a second apart: two standings, two times.
|
|
c.pass(time.Minute)
|
|
first := c.now()
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: first})
|
|
c.pass(time.Second)
|
|
h.Flush() // the runtime's tick, between the two standings
|
|
second := c.now()
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second})
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid {
|
|
t.Fatalf("the restart said again: sends %+v, edits %+v (was %d), replaced %+v", dt.sends[sends:], dt.edits, edits,
|
|
dt.replaced[replaced:])
|
|
}
|
|
if passes(h) != 1 {
|
|
t.Fatalf("the pass ran %d times for one restart", passes(h))
|
|
}
|
|
// The router starts again; the desk repeats its standing.
|
|
again := &Holder{Away: tg, Desk: dt, Store: h.Store, Asks: h.Asks, Bench: h.Bench, Publish: h.Publish, Now: c.now,
|
|
Logf: t.Logf, Zone: h.Zone, RootFree: h.RootFree, Node: h.Node}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
again.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: second})
|
|
runFor(again, c, 2*time.Minute)
|
|
if len(dt.sends) != sends || len(dt.edits) != 0 || len(dt.replaced) != replaced || saidOn(tg) != tgSaid || passes(again) != 1 {
|
|
t.Fatalf("the router's restart made the pass again: %d passes, sends %+v, edits %+v", passes(again),
|
|
dt.sends[sends:], dt.edits)
|
|
}
|
|
}
|
|
|
|
// The controller's walk events are read as ADR 0308 §1 says them; any other plan-moved is not a walk's.
|
|
func TestTheWalkEventsAreReadAsTheControllerSaysThem(t *testing.T) {
|
|
at := time.Date(2026, 10, 12, 11, 12, 0, 0, time.UTC)
|
|
ev, ok, err := DecodeWalk(walkBody(t, "plan-1", WalkStarted, "", at, 1200, novox("mesh-tools", "node-tools")))
|
|
if err != nil || !ok || ev.Walk != "plan-1" || ev.BoundSeconds != 1200 || ev.Class != "leaf" || !ev.At.Equal(at) ||
|
|
len(ev.Touches) != 1 || ev.Touches[0].Machine != "novox" || len(ev.Touches[0].Modules) != 2 {
|
|
t.Fatalf("%+v %v %v", ev, ok, err)
|
|
}
|
|
// As mesh-controller's walk events (its pull request 247) say an end: ended_at, and carried beside modules.
|
|
end, ok, err := DecodeWalk([]byte(`{"id":"plan-1","walk":"plan-1","phase":"ended","outcome":"stopped",` +
|
|
`"touches":[{"machine":"novox","modules":["node-tools","messenger"],"carried":["messenger"]}],` +
|
|
`"started_at":"2026-10-12T11:12:00Z","ended_at":"2026-10-12T11:19:00Z","updated":"2026-10-12T11:19:01Z"}`))
|
|
if err != nil || !ok || end.Outcome != WalkStopped || !end.At.Equal(at.Add(7*time.Minute)) || len(end.Touches[0].Modules) != 2 {
|
|
t.Fatalf("%+v %v %v", end, ok, err)
|
|
}
|
|
if _, ok, err := DecodeWalk([]byte(`{"id":"plan-1","state":"walking","tier":1}`)); ok || err != nil {
|
|
t.Fatalf("a move inside a walk read as its start or end: %v %v", ok, err)
|
|
}
|
|
if _, _, err := DecodeWalk([]byte(`{"walk":"plan-1","phase":"ended"}`)); err == nil {
|
|
t.Fatalf("an end without its outcome was read")
|
|
}
|
|
w := Walk{}
|
|
if w.bound() != DefaultWalkBound {
|
|
t.Fatalf("a walk without a bound is bounded at %s", w.bound())
|
|
}
|
|
// A start older than its bound is history: nothing is held.
|
|
h, _, _, c, _ := newHolder(t)
|
|
walk(t, h, "plan-old", WalkStarted, "", c.now().Add(-time.Hour), 1200, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("an old start opened a walk: %+v", h.walks)
|
|
}
|
|
|
|
}
|
|
|
|
// Review of pull request 230, 1: no channel can send at a walk's end or bound. What it held is not lost with the
|
|
// walk: it is unsent, and said on its own once a channel can send.
|
|
func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) {
|
|
for _, at := range []string{"end", "bound"} {
|
|
t.Run(at, func(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-c", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, 2*time.Minute)
|
|
down := errors.New("no session")
|
|
tg.notReady, dt.notReady = down, down
|
|
if at == "end" {
|
|
walk(t, h, "plan-c", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
} else {
|
|
runFor(h, c, 10*time.Minute)
|
|
}
|
|
if saidOn(tg, dt) != 0 {
|
|
t.Fatalf("said on a channel that cannot send: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if r := h.open[k.Key]; r == nil || r.Pending == "" {
|
|
t.Fatalf("what the walk held is not left unsent: %+v", r)
|
|
}
|
|
tg.notReady, dt.notReady = nil, nil
|
|
runFor(h, c, 2*time.Minute)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("what the walk held was never said: %v", outcomes(h, k.Key))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 2: a message held by walk A, about a machine walk B also touches, is held on at
|
|
// A's end while B — started after it was held — is still open, and told at B's end.
|
|
func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.network", Urgent, "novox lost its default route")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-b", WalkStarted, "", c.now(), 1200, novox("node-tools"))
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-a", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("told at the first walk's end while the second touches it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
if got := h.heldBy(k.Key, c.now()); len(got) != 1 || got[0] != "plan-b" {
|
|
t.Fatalf("the second walk does not hold it on: %v", got)
|
|
}
|
|
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("Still open after it:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("the second walk's end did not tell it: %+v %+v", tg.sends, dt.sends)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 3: a router back after more than a day reads back a walk whose end never came;
|
|
// what it held is said, with the still-running line, before the walk is forgotten.
|
|
func TestAWalkReadBackADayLateSaysWhatItHeldBeforeItIsForgotten(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
walk(t, h, "plan-d", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
c.pass(25 * time.Hour)
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
runFor(again, c, 2*time.Minute)
|
|
if !heard("is still running", dt) || !heard("- Urgent: "+headlineOf(k.Key), dt) {
|
|
t.Fatalf("what the walk held was forgotten unsaid: desk %+v, %v", dt.sends, outcomes(again, k.Key))
|
|
}
|
|
if len(again.walks) != 0 {
|
|
t.Fatalf("the walk was not forgotten: %+v", again.walks)
|
|
}
|
|
if ws, _ := st.Walks(); len(ws) != 0 {
|
|
t.Fatalf("the walk is still kept: %+v", ws)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, 4 (and of mesh-controller pull request 247): a short walk's end can come before
|
|
// its start. The start of a walk already ended opens nothing — also after the router's restart — while the
|
|
// next walk under the same id, started after that end, does.
|
|
func TestAStartHeardAfterItsEndOpensNothing(t *testing.T) {
|
|
h, tg, dt, c, st := newHolder(t)
|
|
started := c.now()
|
|
c.pass(20 * time.Second)
|
|
end, _, err := DecodeWalk([]byte(`{"walk":"plan-e","phase":"ended","outcome":"delivered","touches":[{"machine":"novox","modules":["mesh-tools"]}],` +
|
|
`"started_at":"` + started.Format(time.RFC3339Nano) + `","ended_at":"` + c.now().Format(time.RFC3339Nano) + `"}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.Walked(end)
|
|
c.pass(time.Second)
|
|
walk(t, h, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(h.walks) != 0 {
|
|
t.Fatalf("a start heard after its end opened a walk: %+v", h.walks)
|
|
}
|
|
again := &Holder{Away: tg, Desk: dt, Store: st, Now: c.now, Logf: t.Logf}
|
|
if err := again.Load(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
again.Start()
|
|
walk(t, again, "plan-e", WalkStarted, "", started, 600, novox("mesh-tools"))
|
|
if len(again.walks) != 0 {
|
|
t.Fatalf("after a restart, a start heard after its end opened a walk: %+v", again.walks)
|
|
}
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
again.Condition(EventRaised, k)
|
|
runFor(again, c, time.Minute)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("held by a walk that had ended: %v", outcomes(again, k.Key))
|
|
}
|
|
// Retried: the same id walks again, started after the end.
|
|
walk(t, again, "plan-e", WalkStarted, "", c.now(), 600, novox("mesh-tools"))
|
|
if len(again.walks) != 1 {
|
|
t.Fatalf("a walk started after the last end did not open: %+v", again.walks)
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, non-blocking: a message whose answers became asked while a walk held it is never
|
|
// held (ADR 0308 §3); when the desk's answers are lost, it is said again in place like any other, so its
|
|
// buttons work again.
|
|
func TestARestartSaysAgainAnAskedMessageAWalkTookBeforeItsAsk(t *testing.T) {
|
|
h, _, dt, c, _, _ := newRouter(t)
|
|
h.Start()
|
|
stand(h, c)
|
|
walk(t, h, "plan-f", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
|
|
k := cond("machine.novox.units", Urgent, "2 failed services on novox")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
ask := ackAsk(c, "units")
|
|
ask.About = k.Key
|
|
submit(t, h, ConditionOwner, ask)
|
|
runFor(h, c, time.Minute)
|
|
stand(h, c)
|
|
if !heard(headlineOf(k.Key), dt) {
|
|
t.Fatalf("the ask about it was not offered: %v", outcomes(h, k.Key))
|
|
}
|
|
replaced := len(dt.replaced)
|
|
c.pass(time.Minute)
|
|
h.Stand("desktop", Standing{Ready: true, At: c.now(), AnswersLostAt: c.now()})
|
|
runFor(h, c, time.Minute)
|
|
if len(dt.replaced) != replaced+1 {
|
|
t.Fatalf("its answers were not made to work again: replaced %+v, %v", dt.replaced[replaced:], outcomes(h, "(answers lost)"))
|
|
}
|
|
}
|
|
|
|
// Review of pull request 230, non-blocking: a walk's end that cannot be said as it is (here a module named by a
|
|
// file path) is said in its plain form, and what it held, still open, is said on its own, not marked told.
|
|
func TestWhatAWalksPlainEndCannotCarryIsSaidOnItsOwn(t *testing.T) {
|
|
h, tg, dt, c, _ := newHolder(t)
|
|
walk(t, h, "plan-g", WalkStarted, "", c.now(), 600, novox("/var/lib/mesh-tools"))
|
|
k := cond("machine.novox.tools-silent", Urgent, "the tools on novox are silent")
|
|
k.Raised = c.now()
|
|
h.Condition(EventRaised, k)
|
|
runFor(h, c, time.Minute)
|
|
walk(t, h, "plan-g", WalkEnded, WalkFailed, c.now(), 0)
|
|
runFor(h, c, time.Minute)
|
|
if !heard("A delivery ended", dt) {
|
|
t.Fatalf("the plain end was not said: %+v", dt.sends)
|
|
}
|
|
if !heard(headlineOf(k.Key), tg, dt) {
|
|
t.Fatalf("what the walk held was marked told and never said: %v", outcomes(h, k.Key))
|
|
}
|
|
}
|