Let a drain lapse, bound the gate's wait on the controller's clock, keep a busy wait out of apply time

Review of #283: a drain whose replacement was withdrawn held the agent for
ever with its health green; the gate trusted the engine's until; a machine
of the rest waiting at the bound read as silent and its apply time held the
wait; an unknown mesh-builder word exited 0; plan said nothing of a busy an
older engine is not sent; and a node with a report and no send broke the
reading of every last report (the store test's NULL).

cmd/mesh-controller/busy.go

@@ -21,6 +21,21 @@ import (
 // container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
 var busyGateSlack = 5 * time.Minute
 
+// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
+// controller's clock from when it first read the wait.
+const busyEngineBound = 15 * time.Minute
+
+// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
+// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
+// machine nor a wrong word from it holds a walk longer.
+func busyGateEnds(b *inventory.GateBusy) time.Time {
+	until := b.Seen.Add(busyEngineBound)
+	if !b.Until.IsZero() && b.Until.Before(until) {
+		until = b.Until
+	}
+	return until.Add(busyGateSlack)
+}
+
 // busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
 // true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
 // gate's Since to its end. Pure but for the gate it is handed.
@@ -34,15 +49,19 @@ func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time)
 			continue
 		}
 		if g.Busy == nil {
-			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since}
+			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
+				Seen: now}
+		}
+		if g.Busy.Seen.IsZero() {
+			g.Busy.Seen = now
 		}
 		g.Busy.Until, g.Busy.Said = b.Until, b.Said
-		if now.After(g.Busy.Until.Add(busyGateSlack)) {
+		if now.After(busyGateEnds(g.Busy)) {
 			// The engine said it would have replaced it by now: the judging starts, and finds the module not
 			// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
 			end := now
 			g.Busy.Ended, g.Busy.PastBound = &end, true
-			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt = &end, 0, nil, nil, nil
+			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
 			return false
 		}
 		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
@@ -58,7 +77,7 @@ func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time)
 		end = r.At.UTC()
 	}
 	g.Busy.Ended = &end
-	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt = &end, 0, nil, "", nil, nil
+	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
 	return false
 }