Let a drain lapse, bound the gate's wait on the controller's clock, keep a busy wait out of apply time

Review of #283: a drain whose replacement was withdrawn held the agent for
ever with its health green; the gate trusted the engine's until; a machine
of the rest waiting at the bound read as silent and its apply time held the
wait; an unknown mesh-builder word exited 0; plan said nothing of a busy an
older engine is not sent; and a node with a report and no send broke the
reading of every last report (the store test's NULL).

cmd/mesh-controller/busy.go

@@ -21,6 +21,21 @@ import (
 // container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
 var busyGateSlack = 5 * time.Minute
 
+// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
+// controller's clock from when it first read the wait.
+const busyEngineBound = 15 * time.Minute
+
+// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
+// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
+// machine nor a wrong word from it holds a walk longer.
+func busyGateEnds(b *inventory.GateBusy) time.Time {
+	until := b.Seen.Add(busyEngineBound)
+	if !b.Until.IsZero() && b.Until.Before(until) {
+		until = b.Until
+	}
+	return until.Add(busyGateSlack)
+}
+
 // busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
 // true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
 // gate's Since to its end. Pure but for the gate it is handed.
@@ -34,15 +49,19 @@ func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time)
 			continue
 		}
 		if g.Busy == nil {
-			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since}
+			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
+				Seen: now}
+		}
+		if g.Busy.Seen.IsZero() {
+			g.Busy.Seen = now
 		}
 		g.Busy.Until, g.Busy.Said = b.Until, b.Said
-		if now.After(g.Busy.Until.Add(busyGateSlack)) {
+		if now.After(busyGateEnds(g.Busy)) {
 			// The engine said it would have replaced it by now: the judging starts, and finds the module not
 			// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
 			end := now
 			g.Busy.Ended, g.Busy.PastBound = &end, true
-			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt = &end, 0, nil, nil, nil
+			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
 			return false
 		}
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
@@ -58,7 +77,7 @@ func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time)
 		end = r.At.UTC()
 	}
 	g.Busy.Ended = &end
-	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt = &end, 0, nil, "", nil, nil
+	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
 	return false
 }
 

cmd/mesh-controller/busy_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -109,3 +110,39 @@ func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
 		t.Fatal("a wait reported about an earlier declaration held the gate")
 	}
 }
+
+// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
+// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
+func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
+	seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
+	g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
+	pairs := []judged{{module: "build-agent", node: "novox"}}
+	ahead := seen.Add(time.Hour)
+	f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
+		At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
+			State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
+	if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
+		t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
+	}
+	if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
+		t.Fatal("the gate stopped waiting within the bound on its own clock")
+	}
+	if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
+		t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
+	}
+}
+
+// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
+func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
+	modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
+		{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
+		{"id": "agent-env", "type": "file"}}}}
+	lines := busyNotSent(modules, false)
+	if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
+		!strings.Contains(lines[0], "replaced at once") {
+		t.Fatalf("the plan does not say the busy not sent: %v", lines)
+	}
+	if lines := busyNotSent(modules, true); len(lines) != 0 {
+		t.Fatalf("a busy sent is said as not sent: %v", lines)
+	}
+}

cmd/mesh-controller/plan.go

@@ -1059,6 +1059,25 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string)
 	return had && stated.Contract >= link.BusyContract, nil
 }
 
+// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
+// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
+func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
+	if reads {
+		return nil
+	}
+	var out []string
+	for _, m := range modules {
+		for _, r := range m.Resources {
+			if _, has := r[catalogue.BusyField]; has {
+				out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
+					"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
+					"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
+			}
+		}
+	}
+	return out
+}
+
 // zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
 // 0199): the zone settled from that node's settings, the node's private address, the port the
 // answering listen is published on there.
@@ -1466,6 +1485,15 @@ func planCommand(ctx context.Context, args []string) error {
 	for _, line := range unboundLines(plan) {
 		fmt.Println(line)
 	}
+	// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
+	// container is replaced at once, as before. A statement that cannot be read is said as such.
+	if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
+		fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
+	} else {
+		for _, line := range busyNotSent(plan.Modules, reads) {
+			fmt.Printf("%s: %s\n", args[0], line)
+		}
+	}
 	declared, err := declarationFor(ctx, open, args[0], plan, settings)
 	if err != nil {
 		return err