Never hold a message past its first hold's limit, so a fault on a machine walk after walk touches still reaches the operator (ADR 0505 §7)

This commit is contained in:
2026-10-22 14:36:35 +02:00
parent 34277ef60f
commit 3cf75580b5
6 changed files with 201 additions and 14 deletions
@@ -7,6 +7,8 @@ import (
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The tests of novox/hq ADR 0505 "How it is checked" for the gate: a refusal the gate is already answering is no
@@ -245,3 +247,48 @@ func TestANewHeadWithinTwoHoursKeepsTheRejectedHeadFromTheOperator(t *testing.T)
t.Fatalf("the superseded head reached the operator: %+v", got)
}
}
// ADR 0505 §2: a walk's end said again by the controller with its declared machines (`declared_read`, a touch marked
// declared) acts once: no transition, comment or ask of the controller the first saying did not make.
func TestAWalkSaidAgainWithItsDeclaredMachinesActsOnceInTheGate(t *testing.T) {
w, id, plan := aDeliveringWalk(t)
done := w.ctl.walk(plan)
sent := time.Date(2026, 10, 6, 12, 40, 0, 0, time.UTC)
done.Modules["app"] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &sent, SentAt: &sent, Build: "build-1",
Gate: &WalkGate{Machines: []string{"anchor"}, Since: &sent, Passes: 1, Verdict: "passed", JudgedAt: &sent}}
done.State = walkDone
done = w.ctl.put(done)
body := func(again bool) []byte {
raw, err := json.Marshal(done)
if err != nil {
t.Fatal(err)
}
var said map[string]any
_ = json.Unmarshal(raw, &said)
said["walk"], said["phase"], said["outcome"] = plan, "ended", "delivered"
said["touches"] = []map[string]any{{"machine": "anchor", "modules": []string{"app"}}}
if again {
said["touches"] = append(said["touches"].([]map[string]any),
map[string]any{"machine": "shanks", "modules": []string{}, "declared": true})
said["declared_read"] = true
}
out, _ := json.Marshal(said)
return out
}
if err := take(w.h, stdio.Envelope{Key: ControllerSeat + ".plan-moved", Body: body(false)}); err != nil {
t.Fatal(err)
}
w.settleAll()
if s := w.state(id); s != Delivered {
t.Fatalf("the walk's end left it %s", s)
}
transitions, comments, delivers := len(w.delivery(id).Transitions), len(w.forge.comments), len(w.ctl.delivers)
if err := take(w.h, stdio.Envelope{Key: ControllerSeat + ".plan-moved", Body: body(true)}); err != nil {
t.Fatal(err)
}
w.settleAll()
if len(w.delivery(id).Transitions) != transitions || len(w.forge.comments) != comments || len(w.ctl.delivers) != delivers {
t.Fatalf("said again it acted again: transitions %d→%d, comments %d→%d, asks of the controller %d→%d",
transitions, len(w.delivery(id).Transitions), comments, len(w.forge.comments), delivers, len(w.ctl.delivers))
}
}
@@ -252,7 +252,7 @@ func TestAMemberBothRefusedAndHeldForAStatusIsOneStalledLine(t *testing.T) {
// 1275/AC6, as novox/hq ADR 0505 §4 changed it: a member refused because its verdict is out of date (the trunk moved; its
// fresh check runs) is no stalled line while the gate answers it, and urgent once ten minutes passed with the group not
// merged; the pass stops at the refusal all the same.
func TestAnOutOfDateRefusalIsAWarningAtOnceAndUrgentAfterTenMinutes(t *testing.T) {
func TestAnOutOfDateRefusalIsNoLineWhileAnsweredAndUrgentAfterTenMinutes(t *testing.T) {
w := approvedGroup(t)
allGreen(w)
w.forge.refuse = errAskedAgain
@@ -303,7 +303,7 @@ func severities(w *world) []string {
// 1275/AC6 (b), as ADR 0505 §4 changed it: the same out of date refusal on a LATER member, after the earlier one merged,
// is no line at once and urgent after ten minutes: the half merged group does not say it at once.
func TestAnOutOfDateRefusalOnALaterMemberIsAWarningThenUrgent(t *testing.T) {
func TestAnOutOfDateRefusalOnALaterMemberIsNoLineThenUrgent(t *testing.T) {
w := approvedGroup(t)
allGreen(w)
w.forge.refuseRepo = map[string]error{gCtl: errAskedAgain}
@@ -375,7 +375,7 @@ var mergeToolRefusals = []struct {
// 1275/AC6 (e), as ADR 0505 §4 changed it: only the merge tool's two out of date refusals (:121, :147) are answered by
// the gate and said as no line at once; the one saying its check could not be asked (:144) and every other is urgent at
// once.
func TestOnlyTheMergeToolsTwoOutOfDateRefusalsAreWarnings(t *testing.T) {
func TestOnlyTheMergeToolsTwoOutOfDateRefusalsWaitWhileAnswered(t *testing.T) {
for _, c := range mergeToolRefusals {
if got := outOfDateRefusal(c.words); got != c.outOfDated {
t.Errorf("%s: outOfDateRefusal = %v, wanted %v", c.line, got, c.outOfDated)
@@ -316,3 +316,57 @@ func TestTheDeliveryLineNamesAtMostSixModulesAndNothingTheRuleRefuses(t *testing
t.Fatalf("a refused name in the line: %q", b)
}
}
// ADR 0505 §7: a condition on a machine whose declaration three back-to-back walks change (a machine that stays
// behind is a declared touch of each) reaches the operator at the first walk's bound and settle, while the third
// still runs: never held walk after walk.
func TestADeclaredMachinesConditionAcrossThreeWalksIsSaidAtTheFirstWalksLimit(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
t0 := c.now()
walk(t, h, "plan-1", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared) // limit: t0 + 10 + 2 min
k := cond("machine.shanks.uncomposable", Urgent, "shanks cannot get new instructions")
k.Machine, k.Raised = "shanks", c.now()
h.Condition(EventRaised, k)
runFor(h, c, 5*time.Minute)
walk(t, h, "plan-1", WalkEnded, WalkDelivered, c.now(), 0, novox("gitea"), shanksDeclared)
runFor(h, c, time.Minute)
walk(t, h, "plan-2", WalkStarted, "", c.now(), 600, novox("mesh-issues"), shanksDeclared)
runFor(h, c, 3*time.Minute)
walk(t, h, "plan-2", WalkEnded, WalkDelivered, c.now(), 0, novox("mesh-issues"), shanksDeclared)
runFor(h, c, time.Minute)
walk(t, h, "plan-3", WalkStarted, "", c.now(), 1200, novox("gitea"), shanksDeclared)
for c.now().Before(t0.Add(12*time.Minute - 10*time.Second)) {
runFor(h, c, 5*time.Second)
}
if heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("said before the first walk's limit: %+v", dt.sends)
}
runFor(h, c, 30*time.Second)
if !heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("held past the first walk's bound and settle: %v", outcomes(h, k.Key))
}
}
// ADR 0505 §2: a walk's start or end said again by the controller with its declared machines (declared_read) acts
// once: one walk with both sayings' touches, and one end message.
func TestAWalkSaidAgainWithItsDeclaredMachinesActsOnce(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, novox("gitea"))
walk(t, h, "plan-r", WalkStarted, "", c.now(), 600, novox("gitea"), shanksDeclared)
if len(h.walks) != 1 || len(h.walks["plan-r"].Touches) != 2 {
t.Fatalf("said again it is not one walk with both touches: %+v", h.walks)
}
runFor(h, c, time.Minute)
walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, novox("gitea"))
walk(t, h, "plan-r", WalkEnded, WalkFailed, c.now(), 0, novox("gitea"), shanksDeclared)
runFor(h, c, 3*time.Minute)
n := 0
for _, m := range append(append([]Message(nil), dt.sends...), tg.sends...) {
if strings.HasPrefix(m.Title, "Delivery failed") {
n++
}
}
if n != 1 {
t.Fatalf("its end was told %d times: %+v", n, dt.sends)
}
}
@@ -39,6 +39,11 @@ func (h *Holder) deliver(r *Record, what string) {
// An open walk touches it: told when the walk ends or passes its bound (ADR 0308 §2).
return
}
h.route(r, what)
}
// route sends a record's message where its severity routes it, held by no walk.
func (h *Holder) route(r *Record, what string) {
var to []Channel
if kinds := h.askKinds(r); len(kinds) > 0 {
// An ask goes to every channel that carries one of its answers (novox/hq ADR 0259): the desk for an
+85 -4
View File
@@ -125,8 +125,15 @@ type WalkEvent struct {
type HeldMessage struct {
At time.Time `json:"at"`
What string `json:"what"`
// Until is the first hold's limit: the start, bound and settle of the walk that first held it, or that walk's
// settle's end when it was held settling (ADR 0505 §7). Kept when it is handed on: no walk holds it past this, so a
// fault on a machine walk after walk touches reaches the operator all the same. Zero in a hold kept before it was.
Until time.Time `json:"until,omitzero"`
}
// past says whether a held message's first hold's limit has passed.
func (hm HeldMessage) past(now time.Time) bool { return !hm.Until.IsZero() && !now.Before(hm.Until) }
// Walk is one open walk as this router keeps it.
type Walk struct {
ID string `json:"id"`
@@ -164,6 +171,15 @@ func (w *Walk) settleFor() time.Duration { return time.Duration(w.SettleSeconds)
// limit is the longest a walk holds anything: its start, its bound and its settle (ADR 0505 §7).
func (w *Walk) limit() time.Time { return w.Started.Add(w.bound() + w.settleFor()) }
// holdEnd is the last moment the walk holds anything it holds now: its limit, or its settle's end when that is
// sooner.
func (w *Walk) holdEnd() time.Time {
if w.settling() && w.SettleUntil.Before(w.limit()) {
return w.SettleUntil
}
return w.limit()
}
// settling says whether the walk ended delivered and holds on until its settle has passed (ADR 0505 §3).
func (w *Walk) settling() bool { return !w.SettleUntil.IsZero() }
@@ -483,11 +499,12 @@ func (h *Holder) release(w *Walk, now time.Time) (open []*Record, whats map[stri
if o.ID == w.ID || !o.holding(now) {
continue
}
if _, ok := o.Held[key]; ok {
elsewhere = true
if other, ok := o.Held[key]; ok {
elsewhere = elsewhere || !other.past(now)
continue
}
if live && o.touches(s) {
if live && o.touches(s) && !hm.past(now) {
// Handed on with its first hold's limit: never held past it (ADR 0505 §7).
if o.Held == nil {
o.Held = map[string]HeldMessage{}
}
@@ -561,6 +578,7 @@ func (h *Holder) endWalk(w *Walk, outcome string, ended, now time.Time) {
// settledWalks tells the end of every delivered walk whose settle has passed (ADR 0505 §3), oldest first.
func (h *Holder) settledWalks(now time.Time) {
h.overdueHolds(now)
h.mu.Lock()
var settled []*Walk
for _, w := range h.walks {
@@ -575,6 +593,49 @@ func (h *Holder) settledWalks(now time.Time) {
}
}
// overdueHolds says every held message whose first hold's limit has passed while a walk that took it over still holds,
// on its own, with its delivery line (ADR 0505 §7): no walk holds it past that limit. A walk whose own end or bound is
// due tells what it holds in its message instead. Called at each flush.
func (h *Holder) overdueHolds(now time.Time) {
h.mu.Lock()
due := map[string]HeldMessage{}
var touched []*Walk
for _, w := range h.walks {
changed := false
if !now.Before(w.holdEnd()) {
continue // its own end or bound tells what it holds, in its message
}
for key, hm := range w.Held {
if hm.past(now) {
if prev, ok := due[key]; !ok || hm.At.Before(prev.At) {
due[key] = hm
}
delete(w.Held, key)
changed = true
}
}
if changed {
touched = append(touched, w)
}
}
var recs []*Record
for key, hm := range due {
if r := h.open[key]; r != nil && r.Cleared.IsZero() && !r.silenced(now) && !lastSaid(r).After(hm.At) {
recs = append(recs, r)
}
}
h.mu.Unlock()
for _, w := range touched {
h.keepWalk(w)
}
sort.Slice(recs, func(i, j int) bool { return recs[i].Key < recs[j].Key })
for _, r := range recs {
what := due[r.Key].What
h.record(Sent{At: now, Channel: "-", Key: r.Key, What: what, Outcome: "released: past its first hold's limit"})
h.route(r, what)
}
}
// endSettled tells the end of a delivered walk whose settle has passed (ADR 0505 §3): what it held still open, one
// line each, and the cleared counted; then it is only the tomb that keeps a late start from opening it again.
func (h *Holder) endSettled(w *Walk, now time.Time) {
@@ -964,6 +1025,26 @@ func (h *Holder) hold(r *Record, what string) bool {
Outcome: "not held: its walk " + f.Outcome})
return false
}
// The first hold's limit: kept by whichever walk holds it now, else the soonest end of the walks about to.
var until time.Time
for _, w := range h.walks {
if hm, ok := w.Held[r.Key]; ok && !hm.Until.IsZero() && (until.IsZero() || hm.Until.Before(until)) {
until = hm.Until
}
}
if until.IsZero() {
for _, w := range h.walks {
if w.holding(now) && w.touches(s) && (until.IsZero() || w.holdEnd().Before(until)) {
until = w.holdEnd()
}
}
}
if !until.IsZero() && !now.Before(until) {
// Held to its first hold's limit already: said now, with its delivery line (ADR 0505 §7).
h.mu.Unlock()
h.record(Sent{At: now, Channel: "-", Key: r.Key, What: what, Outcome: "not held: past its first hold's limit"})
return false
}
var by []*Walk
var ids []string
fresh := false
@@ -972,7 +1053,7 @@ func (h *Holder) hold(r *Record, what string) bool {
continue
}
prev, had := w.Held[r.Key]
hm := HeldMessage{At: now, What: what}
hm := HeldMessage{At: now, What: what, Until: until}
if had {
hm.At = prev.At
if !r.told() && prev.What == "raised" {
@@ -480,11 +480,12 @@ func TestWhatAWalkHeldIsSaidOnceAChannelCanSendAgain(t *testing.T) {
}
}
// Review of pull request 230, 2: a message held by walk A, about a machine walk B also touches, is held on at
// A's end while B — started after it was held — is still open, and told at B's end.
// Review of pull request 230, 2, as novox/hq ADR 0505 §7 bounds it: a message held by walk A, about a machine walk B
// also touches, is held on at A's end while B — started after it was held — is still open, and is said once A's own
// limit (its start, bound and settle) has passed, while B still runs, with B's delivery line.
func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
h, tg, dt, c, _ := newHolder(t)
walk(t, h, "plan-a", WalkStarted, "", c.now(), 1200, novox("mesh-tools"))
walk(t, h, "plan-a", WalkStarted, "", c.now(), 360, novox("mesh-tools")) // its limit: 6 min and the settle
k := cond("machine.novox.network", Urgent, "novox lost its default route")
k.Raised = c.now()
h.Condition(EventRaised, k)
@@ -499,10 +500,9 @@ func TestAMessageIsHeldOnByALaterWalkThatTouchesIt(t *testing.T) {
if got := h.heldBy(k.Key, c.now()); len(got) != 1 || got[0] != "plan-b" {
t.Fatalf("the second walk does not hold it on: %v", got)
}
walk(t, h, "plan-b", WalkEnded, WalkDelivered, c.now(), 0)
runFor(h, c, DefaultSettle+time.Minute)
if !heard("Still open after the delivery of node-tools to novox:\n- Urgent: "+headlineOf(k.Key), tg, dt) {
t.Fatalf("the second walk's end did not tell it: %+v %+v", tg.sends, dt.sends)
runFor(h, c, 4*time.Minute) // past the first walk's limit, 8 minutes after it was held
if !heard("During the delivery of node-tools to novox.", tg, dt) || !heard(headlineOf(k.Key), tg, dt) {
t.Fatalf("held past its first hold's limit: %+v %+v", tg.sends, dt.sends)
}
}