Let a drain lapse, bound the gate's wait on the controller's clock, keep a busy wait out of apply time

Review of #283: a drain whose replacement was withdrawn held the agent for
ever with its health green; the gate trusted the engine's until; a machine
of the rest waiting at the bound read as silent and its apply time held the
wait; an unknown mesh-builder word exited 0; plan said nothing of a busy an
older engine is not sent; and a node with a report and no send broke the
reading of every last report (the store test's NULL).

cmd/mesh-builder/busy.go

@@ -28,10 +28,49 @@ import (
 func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
 func drainMark() string  { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
 
-// drained says the agent was asked whether it is busy: it takes no new build request.
-func drained() bool {
-	_, err := os.Stat(drainMark())
-	return err == nil
+// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
+// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
+// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
+// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
+// came, which is a fault to see.
+const (
+	drainLapse   = 2 * time.Minute
+	drainTooLong = 15*time.Minute + 5*time.Minute
+)
+
+// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
+func drainedAt() (began, asked time.Time, ok bool) {
+	info, err := os.Stat(drainMark())
+	if err != nil {
+		return time.Time{}, time.Time{}, false
+	}
+	asked = info.ModTime()
+	began = asked
+	if raw, err := os.ReadFile(drainMark()); err == nil {
+		if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
+			began = t
+		}
+	}
+	return began, asked, true
+}
+
+// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
+func drained() bool { return drainedSince(time.Now()) }
+
+func drainedSince(now time.Time) bool {
+	_, asked, ok := drainedAt()
+	return ok && now.Sub(asked) < drainLapse
+}
+
+// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
+// longer than its node-engine's bound and the slack.
+func drainedTooLong(now time.Time) error {
+	began, asked, ok := drainedAt()
+	if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
+		return nil
+	}
+	return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
+		"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
 }
 
 // claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
@@ -85,7 +124,13 @@ func undrained() error {
 
 // busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
 func busy() (string, error) {
-	if err := os.WriteFile(drainMark(), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+	// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
+	// ask: its time is the last ask.
+	began := time.Now().UTC()
+	if was, _, ok := drainedAt(); ok && drained() {
+		began = was.UTC()
+	}
+	if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
 		// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
 		fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
 	}

cmd/mesh-builder/busy_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"os"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 )
 
 // `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
@@ -75,3 +77,86 @@ func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
 		released()
 	}
 }
+
+// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
+// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
+func TestADrainNoLongerAskedLapses(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	if !drained() || claim() {
+		t.Fatal("an agent just asked takes work")
+	}
+	// The last ask was three minutes ago.
+	old := time.Now().Add(-3 * time.Minute)
+	if err := os.Chtimes(drainMark(), old, old); err != nil {
+		t.Fatal(err)
+	}
+	if drained() {
+		t.Fatal("a drain no longer asked for three minutes still holds")
+	}
+	if !claim() {
+		t.Fatal("an agent whose drain lapsed takes no work")
+	}
+	released()
+}
+
+// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
+// replacement it was drained for never came. One asked again after a lapse starts a new drain.
+func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	taking("mesh-build-machine")
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	now := time.Now()
+	if err := drainedTooLong(now); err != nil {
+		t.Fatalf("an agent drained a moment ago is not ready: %v", err)
+	}
+	// The drain began 25 minutes ago, and the node-engine asked a moment ago.
+	began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
+	if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	err := drainedTooLong(time.Now())
+	if err == nil || !strings.Contains(err.Error(), "has not come") {
+		t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
+	}
+	// Each ask keeps when the drain began.
+	if _, err := busy(); err == nil {
+		t.Fatal("an idle agent answered busy")
+	}
+	if err := drainedTooLong(time.Now()); err == nil {
+		t.Fatal("an ask started the drain again")
+	}
+	// The asks stopped: the drain lapsed, and so did the fault.
+	old := time.Now().Add(-3 * time.Minute)
+	if err := os.Chtimes(drainMark(), old, old); err != nil {
+		t.Fatal(err)
+	}
+	if err := drainedTooLong(time.Now()); err != nil {
+		t.Fatalf("a lapsed drain still reads not ready: %v", err)
+	}
+}
+
+// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
+// node-engine's whole bound.
+func TestAnUnknownCommandFails(t *testing.T) {
+	was := os.Args
+	t.Cleanup(func() { os.Args = was })
+	os.Args = []string{"mesh-builder", "some-later-word"}
+	if err := run(); err == nil {
+		t.Fatal("an unknown command exited 0")
+	}
+	os.Args = []string{"mesh-builder", "help"}
+	if err := run(); err != nil {
+		t.Fatalf("help failed: %v", err)
+	}
+}

cmd/mesh-builder/main.go

@@ -92,8 +92,12 @@ func run() error {
 		case "build":
 			return buildOnce(context.Background(), os.Args[2:])
 		case "ready":
-			// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
-			return ready()
+			// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
+			// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
+			if err := ready(); err != nil {
+				return err
+			}
+			return drainedTooLong(time.Now())
 		case "busy":
 			// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
 			// nothing is in hand.
@@ -103,9 +107,14 @@ func run() error {
 			}
 			fmt.Println(said)
 			return nil
-		default:
+		case "help", "-h", "--help":
 			fmt.Print(usage)
 			return nil
+		default:
+			// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
+			// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
+			fmt.Fprint(os.Stderr, usage)
+			return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
 		}
 	}
 

cmd/mesh-controller/busy.go

@@ -21,6 +21,21 @@ import (
 // container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
 var busyGateSlack = 5 * time.Minute
 
+// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
+// controller's clock from when it first read the wait.
+const busyEngineBound = 15 * time.Minute
+
+// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
+// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
+// machine nor a wrong word from it holds a walk longer.
+func busyGateEnds(b *inventory.GateBusy) time.Time {
+	until := b.Seen.Add(busyEngineBound)
+	if !b.Until.IsZero() && b.Until.Before(until) {
+		until = b.Until
+	}
+	return until.Add(busyGateSlack)
+}
+
 // busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
 // true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
 // gate's Since to its end. Pure but for the gate it is handed.
@@ -34,15 +49,19 @@ func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time)
 			continue
 		}
 		if g.Busy == nil {
-			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since}
+			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
+				Seen: now}
+		}
+		if g.Busy.Seen.IsZero() {
+			g.Busy.Seen = now
 		}
 		g.Busy.Until, g.Busy.Said = b.Until, b.Said
-		if now.After(g.Busy.Until.Add(busyGateSlack)) {
+		if now.After(busyGateEnds(g.Busy)) {
 			// The engine said it would have replaced it by now: the judging starts, and finds the module not
 			// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
 			end := now
 			g.Busy.Ended, g.Busy.PastBound = &end, true
-			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt = &end, 0, nil, nil, nil
+			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
 			return false
 		}
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
@@ -58,7 +77,7 @@ func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time)
 		end = r.At.UTC()
 	}
 	g.Busy.Ended = &end
-	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt = &end, 0, nil, "", nil, nil
+	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
 	return false
 }
 

cmd/mesh-controller/busy_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -109,3 +110,39 @@ func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
 		t.Fatal("a wait reported about an earlier declaration held the gate")
 	}
 }
+
+// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
+// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
+func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
+	seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
+	pairs := []judged{{module: "build-agent", node: "novox"}}
+	ahead := seen.Add(time.Hour)
+	f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
+		At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
+			State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
+	if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
+		t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
+	}
+	if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
+		t.Fatal("the gate stopped waiting within the bound on its own clock")
+	}
+	if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
+		t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
+	}
+}
+
+// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
+func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
+	modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
+		{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
+		{"id": "agent-env", "type": "file"}}}}
+	lines := busyNotSent(modules, false)
+	if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
+		!strings.Contains(lines[0], "replaced at once") {
+		t.Fatalf("the plan does not say the busy not sent: %v", lines)
+	}
+	if lines := busyNotSent(modules, true); len(lines) != 0 {
+		t.Fatalf("a busy sent is said as not sent: %v", lines)
+	}
+}

cmd/mesh-controller/plan.go

@@ -1059,6 +1059,25 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string)
 	return had && stated.Contract >= link.BusyContract, nil
 }
 
+// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
+// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
+func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
+	if reads {
+		return nil
+	}
+	var out []string
+	for _, m := range modules {
+		for _, r := range m.Resources {
+			if _, has := r[catalogue.BusyField]; has {
+				out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
+					"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
+					"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
+			}
+		}
+	}
+	return out
+}
+
 // zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
 // 0199): the zone settled from that node's settings, the node's private address, the port the
 // answering listen is published on there.
@@ -1466,6 +1485,15 @@ func planCommand(ctx context.Context, args []string) error {
 	for _, line := range unboundLines(plan) {
 		fmt.Println(line)
 	}
+	// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
+	// container is replaced at once, as before. A statement that cannot be read is said as such.
+	if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
+		fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
+	} else {
+		for _, line := range busyNotSent(plan.Modules, reads) {
+			fmt.Printf("%s: %s\n", args[0], line)
+		}
+	}
 	declared, err := declarationFor(ctx, open, args[0], plan, settings)
 	if err != nil {
 		return err