Hold a walk's first-machine gate while its busy container finishes its build

A merge into mesh-controller replaced every build agent and threw away the
checks in flight (issue 594). The manifest's container busy is checked and
sent only to an engine that reads it; a report's busy waits are kept, the
gate starts at the report that says the container replaced, and the wait is
a phase of the delivery time. mesh-builder busy drains the agent and answers
while a build is in hand (novox/hq ADR 0335).

cmd/mesh-controller/busy.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
+//
+// When a machine reports, on what the gate sent it, that a container of a judged module was left running
+// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
+// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
+// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
+// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
+// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
+// its gate starts as before.
+
+// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
+// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
+var busyGateSlack = 5 * time.Minute
+
+// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
+// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
+// gate's Since to its end. Pure but for the gate it is handed.
+func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
+	if g.Busy != nil && g.Busy.Ended != nil {
+		return false
+	}
+	for _, j := range pairs {
+		b, waits := busyOn(j, f)
+		if !waits {
+			continue
+		}
+		if g.Busy == nil {
+			g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since}
+		}
+		g.Busy.Until, g.Busy.Said = b.Until, b.Said
+		if now.After(g.Busy.Until.Add(busyGateSlack)) {
+			// The engine said it would have replaced it by now: the judging starts, and finds the module not
+			// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
+			end := now
+			g.Busy.Ended, g.Busy.PastBound = &end, true
+			g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt = &end, 0, nil, nil, nil
+			return false
+		}
+		g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
+		g.Read(now, false, g.Busy.Words())
+		return true
+	}
+	if g.Busy == nil {
+		return false
+	}
+	// Replaced: the judging starts at the report that said so.
+	end := now
+	if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
+		end = r.At.UTC()
+	}
+	g.Busy.Ended = &end
+	g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt = &end, 0, nil, "", nil, nil
+	return false
+}
+
+// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
+// running because it said busy.
+func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
+	r, said := f.reports[j.node]
+	if !said || r.At == nil || !f.reportedOn(j.node, r) {
+		return inventory.BusyWait{}, false
+	}
+	return inventory.WaitingFor(r.Busy, j.module)
+}
+
+// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
+// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
+func busyStill(j judged, f gateFacts) (string, bool) {
+	b, waits := busyOn(j, f)
+	if !waits {
+		return "", false
+	}
+	return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
+		j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
+}