Leave a busy container running until it says idle, within 15 minutes

A send that replaced the build agent killed the check it ran, and the queue
started it again elsewhere (issue 594). A container may now declare busy; the
node-engine asks it before replacing it, applies everything else, says the
wait, and replaces it on the first idle pass or at the bound, kept across
restarts (novox/hq ADR 0335).

internal/declaration/busy.go

@@ -0,0 +1,87 @@
+package declaration
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// Busy is how a long-running container says it holds work that would be lost if it were stopped now
+// (novox/hq ADR 0335 §1): a command run inside it, by its shell, as an exec health check is, that exits 0
+// while it is busy and non-zero when it is idle. Being asked is also the container's notice that it is
+// about to be replaced: from the first ask it takes no new work, so its answer ends once the work in hand
+// ends.
+//
+// **Not part of what the container is.** Busy is not in the container's spec: a changed command changes
+// when the container is replaced, never whether — a container is not recreated for its busy command alone.
+type Busy struct {
+	Command string `json:"command"`
+	// Timeout is how long one ask may take; empty is BusyTimeoutDefault.
+	Timeout string `json:"timeout,omitempty"`
+}
+
+// The bounds of an ask (novox/hq ADR 0335).
+const (
+	BusyTimeoutDefault = 10 * time.Second
+	// BusyTimeoutCeiling is the longest one ask may take: an ask runs inside an apply, and an apply waits
+	// for nothing longer than a look.
+	BusyTimeoutCeiling = time.Minute
+	// BusyBound is how long a container that says busy is left running after the first apply that would
+	// have replaced it (ADR 0335 §3). The node-engine's, not the module's: no module holds its own
+	// replacement for ever.
+	BusyBound = 15 * time.Minute
+)
+
+// UnmarshalJSON reads a busy strictly, as everything a declaration carries is read.
+func (b *Busy) UnmarshalJSON(raw []byte) error {
+	type plain Busy
+	var p plain
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&p); err != nil {
+		return fmt.Errorf("busy: %w", err)
+	}
+	*b = Busy(p)
+	return nil
+}
+
+// Within is how long one ask may take. Validated on arrival, so an unreadable timeout is the default.
+func (b *Busy) Within() time.Duration {
+	if d, err := time.ParseDuration(b.Timeout); err == nil && d > 0 {
+		return d
+	}
+	return BusyTimeoutDefault
+}
+
+// problems holds a container's busy to its shape: a one-line command, a timeout within its bounds, and only
+// on a container that stays up.
+func (b *Busy) problems(where string, longRunning bool) []string {
+	if b == nil {
+		return nil
+	}
+	var problems []string
+	say := func(format string, args ...any) {
+		problems = append(problems, where+": "+fmt.Sprintf(format, args...))
+	}
+	if !longRunning {
+		say("busy is asked of a container that stays up; a step or a scheduled run is never replaced mid-run")
+	}
+	if strings.TrimSpace(b.Command) == "" {
+		say("busy needs a command")
+	}
+	if strings.ContainsAny(b.Command, "\n\r") {
+		say("busy's command is one line")
+	}
+	if b.Timeout != "" {
+		d, err := time.ParseDuration(b.Timeout)
+		switch {
+		case err != nil:
+			say("busy's timeout is a duration, not %q", b.Timeout)
+		case d <= 0 || d > BusyTimeoutCeiling:
+			say("an ask of busy takes more than nothing and at most %s, not %s", BusyTimeoutCeiling, d)
+		}
+	}
+	return problems
+}

internal/declaration/declaration.go

@@ -1239,6 +1239,11 @@ type Container struct {
 	// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
 	Health *Health `json:"health,omitempty"`
 
+	// Busy is how this container says it holds work a replacement would lose (novox/hq ADR 0335): asked
+	// before it is replaced, and while it says busy it is left running, within the bound. Absent: never
+	// busy, and replaced at once, as before.
+	Busy *Busy `json:"busy,omitempty"`
+
 	// RunAs is who the container runs as, resolved by the controller from the module's identity and the
 	// image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and
 	// PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the
@@ -1318,6 +1323,7 @@ func (c *Container) Target() string   { return c.Name }
 
 func (c *Container) validate(where string, _ bool) []string {
 	problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "")
+	problems = append(problems, c.Busy.problems(where, !c.RunOnce && c.Schedule == "")...)
 	if c.Name == "" {
 		problems = append(problems, where+": a container needs a name")
 	}