Hold a walk's first-machine gate while its busy container finishes its build

A merge into mesh-controller replaced every build agent and threw away the
checks in flight (issue 594). The manifest's container busy is checked and
sent only to an engine that reads it; a report's busy waits are kept, the
gate starts at the report that says the container replaced, and the wait is
a phase of the delivery time. mesh-builder busy drains the agent and answers
while a build is in hand (novox/hq ADR 0335).

cmd/mesh-builder/busy.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
+// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
+// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
+// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
+// it runs, instead of the build queue handing it out again from the start (issue 594).
+//
+// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
+// without them, and a restarted one removes them before it takes work again.
+//
+//   - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
+//     when the build ends.
+//   - drained: written by the ask, before it reads whether a build is in hand.
+//
+// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
+// or the agent sees the drain and hands the request back; never neither.
+
+func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
+func drainMark() string  { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
+
+// drained says the agent was asked whether it is busy: it takes no new build request.
+func drained() bool {
+	_, err := os.Stat(drainMark())
+	return err == nil
+}
+
+// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
+// back to the queue.
+func claim() bool {
+	// Made, never written over: one build at a time, and a mark already there is a build in hand.
+	f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+	if err != nil {
+		// Unmarked, an ask would say idle while this build runs: the build is not taken.
+		fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
+		return false
+	}
+	_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
+	if closeErr := f.Close(); err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		_ = os.Remove(inHandMark())
+		fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
+		return false
+	}
+	if drained() {
+		_ = os.Remove(inHandMark())
+		fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
+		return false
+	}
+	return true
+}
+
+// holding says what is in hand, for the ask to print: written once the request is read.
+func holding(what string) {
+	_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
+}
+
+// released ends the build in hand.
+func released() {
+	if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
+		fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
+	}
+}
+
+// undrained clears both marks at the start of a holder: a container started again is asked afresh.
+func undrained() error {
+	for _, mark := range []string{drainMark(), inHandMark()} {
+		if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("cannot remove %s: %w", mark, err)
+		}
+	}
+	return nil
+}
+
+// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
+func busy() (string, error) {
+	if err := os.WriteFile(drainMark(), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+		// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
+		fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
+	}
+	held, err := os.ReadFile(inHandMark())
+	if err != nil {
+		return "", errors.New("idle: no build in hand, and this agent takes no new one")
+	}
+	since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
+	if what == "" {
+		what = "a build"
+	}
+	return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
+}

cmd/mesh-builder/busy_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"strings"
+	"sync"
+	"testing"
+)
+
+// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
+// agent takes no new build request (novox/hq ADR 0335 §5).
+func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	if drained() {
+		t.Fatal("an agent never asked is drained")
+	}
+	// A build is claimed and runs.
+	if !claim() {
+		t.Fatal("an agent never asked refused a build")
+	}
+	holding("novox/mesh-controller (check-280)")
+	said, err := busy()
+	if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
+		t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
+	}
+	// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
+	if !drained() {
+		t.Fatal("the ask did not drain the agent")
+	}
+	if claim() {
+		t.Fatal("a drained agent took a new build")
+	}
+	// The build in hand is still in hand after the refused claim, and busy goes on saying so.
+	if _, err := busy(); err != nil {
+		t.Fatalf("a refused claim ended the build in hand: %v", err)
+	}
+	// It ends: idle, and still drained.
+	released()
+	if _, err := busy(); err == nil {
+		t.Fatal("an agent with nothing in hand answered busy")
+	}
+	if claim() {
+		t.Fatal("a drained agent took a build after its last one ended")
+	}
+	// Started again, it is asked afresh.
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	if drained() || !claim() {
+		t.Fatal("a holder started again is still drained")
+	}
+	released()
+}
+
+// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
+// neither, which would replace the container under a build just taken.
+func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	for i := 0; i < 200; i++ {
+		if err := undrained(); err != nil {
+			t.Fatal(err)
+		}
+		var claimed bool
+		var askErr error
+		var wg sync.WaitGroup
+		wg.Add(2)
+		go func() { defer wg.Done(); claimed = claim() }()
+		go func() { defer wg.Done(); _, askErr = busy() }()
+		wg.Wait()
+		if claimed && askErr != nil {
+			t.Fatalf("round %d: the agent took a build and the ask said idle", i)
+		}
+		released()
+	}
+}

cmd/mesh-builder/main.go

@@ -71,6 +71,12 @@ digest of their own configuration. The result is printed as JSON.
 
 exits 0 once the agent running in this container is bound to its build work and pulls it, and
 1 before: its container's health check.
+
+  mesh-builder busy
+
+exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
+from the first ask the agent takes no new build: its container's busy command, asked before the
+container is replaced (novox/hq ADR 0335).
 `
 
 func run() error {
@@ -84,6 +90,15 @@ func run() error {
 		case "ready":
 			// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
 			return ready()
+		case "busy":
+			// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
+			// nothing is in hand.
+			said, err := busy()
+			if err != nil {
+				return err
+			}
+			fmt.Println(said)
+			return nil
 		default:
 			fmt.Print(usage)
 			return nil
@@ -93,6 +108,9 @@ func run() error {
 	if err := notReady(); err != nil {
 		return err
 	}
+	if err := undrained(); err != nil {
+		return err
+	}
 	credential, err := brokerFrom()
 	if err != nil {
 		return err
@@ -145,8 +163,13 @@ func run() error {
 	// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
 	// recreated mid-check left the check's store and bus running, and the ask went to another machine.
 	h.removeLeftBehind()
-	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
-		Taking: func() { taking(seat) }})
+	// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
+	// in hand and then says idle, and its container is replaced between builds.
+	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
+		Paused: func() bool { return h.Paused() || drained() },
+		Taking: func() { taking(seat) },
+		Claim:  claim,
+	})
 	defer machine.Close()
 
 	// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
@@ -172,6 +195,8 @@ func run() error {
 	publisher := builder.Registry{Address: registry, Run: builder.Command}
 
 	return machine.Take(ctx, func(ctx context.Context, work link.Build) {
+		defer released()
+		holding(work.Request().Repository + " (" + work.Request().ID + ")")
 		answer(ctx, publisher, on, workspace, work, h)
 	})
 }

cmd/mesh-builder/stdout_test.go

@@ -16,6 +16,7 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
 		"version)":     true, // --version
 		`"stopping")`:  true, // the loop.s shutdown line
 		"usage)":       true, // --help text, for a human
+		"said)":        true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
 	}
 	for _, file := range []string{"once.go", "main.go"} {
 		src, err := os.ReadFile(file)