Leave a busy container running until it says idle, within 15 minutes

A send that replaced the build agent killed the check it ran, and the queue
started it again elsewhere (issue 594). A container may now declare busy; the
node-engine asks it before replacing it, applies everything else, says the
wait, and replaces it on the first idle pass or at the bound, kept across
restarts (novox/hq ADR 0335).

internal/apply/apply.go

@@ -92,6 +92,11 @@ type Outcome struct {
 	// reads is, for a container, the digest of each file it was created reading, by path — so
 	// the next apply can say which one changed (novox/hq 04-ISSUES/103).
 	reads map[string]string
+	// busySince is, for a container left running because it said busy, when the first apply that would
+	// have replaced it ran (novox/hq ADR 0335): kept in its record, so the bound holds across passes and
+	// across a restart of the node-engine. busy is what the report says of it.
+	busySince *time.Time
+	busy      *BusyWait
 }
 
 // Report is what an apply did, in the order it did it.
@@ -108,6 +113,20 @@ type Report struct {
 	// 224): a machine whose server is stopped at 03:31 because its collector is running is
 	// working, not broken, and the report is where that difference is said.
 	Windows []Window `json:"windows,omitempty"`
+	// Busy is every container this apply left running because it said busy, or replaced after it had
+	// waited (novox/hq ADR 0335): a machine whose walk waits for it says why, and how it ended.
+	Busy []BusyWait `json:"busy,omitempty"`
+}
+
+// Waiting says whether a container of this apply waits to be replaced because it is busy: what makes the
+// node-engine ask again sooner than its reconcile (novox/hq ADR 0335 §2).
+func (r Report) Waiting() bool {
+	for _, b := range r.Busy {
+		if b.State == BusyWaiting {
+			return true
+		}
+	}
+	return false
 }
 
 // Changed reports whether anything about the machine actually moved. An apply that changed
@@ -118,7 +137,10 @@ func (r Report) Changed() bool {
 		// whose account's manager is not running, which nothing moved either (novox/hq ADR 0177);
 		// held still is a container a maintenance window stopped, which the apply did not touch
 		// (novox/hq issue 224).
-		if o.Action != "unchanged" && o.Action != "held" && o.Action != "waiting" && o.Action != heldStill {
+		// waits busy is a container left running because it said busy (novox/hq ADR 0335), which nothing
+		// moved either.
+		if o.Action != "unchanged" && o.Action != "held" && o.Action != "waiting" && o.Action != heldStill &&
+			o.Action != waitsBusy {
 			return true
 		}
 	}
@@ -987,6 +1009,7 @@ func ApplyMindingWindows(
 			KeptMode:     keptMode,
 			KeptOwner:    keptOwner,
 			Reads:        outcome.reads,
+			BusySince:    outcome.busySince,
 			Stateless:    outcome.stateless,
 			Scope:        outcome.scope,
 			User:         outcome.user,
@@ -1046,7 +1069,10 @@ func ApplyMindingWindows(
 				Action: "forgotten", Detail: detail})
 			log(fmt.Sprintf("  forgotten %s (%s): %s", from.ID, from.Target, detail))
 		}
-		if outcome.Action == heldStill {
+		if outcome.busy != nil {
+			report.Busy = append(report.Busy, *outcome.busy)
+		}
+		if outcome.Action == heldStill || outcome.Action == waitsBusy {
 			// Not changed: nothing about it moved, so nothing that restarts on it restarts.
 			log(fmt.Sprintf("  %s %s (%s): %s", outcome.Action, outcome.ID, outcome.Target, outcome.Detail))
 		} else if outcome.Action != "unchanged" {
@@ -2707,6 +2733,15 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 		out.Action = "unchanged"
 		return out, nil
 	case existed:
+		// **A container that says it is busy is asked before it is replaced** (novox/hq ADR 0335 §2): left
+		// running while it says busy, within the bound, and replaced on the first pass it says idle. Only a
+		// running one: a stopped container holds no work in hand.
+		if r.Busy != nil && before.Running {
+			if waits := askBeforeReplacing(ctx, r, run, cri, previous, &out); waits {
+				out.reads = wasReading
+				return out, nil
+			}
+		}
 		if _, err := run(ctx, cri, "rm", "-f", r.Name); err != nil {
 			return out, fmt.Errorf("replacing container %s: %w", r.Name, err)
 		}
@@ -2807,6 +2842,9 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
 		default:
 			out.Detail = "replaced; a container's configuration is fixed when it is created"
 		}
+		if out.busy != nil {
+			out.Detail += "; " + out.busy.Words()
+		}
 	}
 	return out, nil
 }