Two machines holding the build role share one queue, and neither is handed an ask while busy (hq ADR 0190)

Against a real bus: three asks, two machines; each takes one, the third waits until one is free
and then goes to that one; a machine that stops leaves nothing taken twice. The redelivery of an
ask a dead machine held is the ack wait's, proven by the hand-back test beside this one.

And the order a live mesh switches over in, written where the role is named: queued builds first,
then this controller, then build-agent assigned where machines build, then the builder and the old
seat's stream forgotten.

internal/link/builds.go

@@ -22,6 +22,13 @@ import (
 // TheBuildMachine is the role a build is submitted to: node-scoped, held on every machine that
 // builds, and the work shared among them (novox/hq ADR 0190). The name stays for every caller; what
 // it names moved from the mesh's one build machine to whichever build agent is idle.
+//
+// **Switching a live mesh over, in order.** The old seat's stream and worker
+// (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until removed by hand,
+// and the builder module keeps draining them while it is assigned. From the moment a controller
+// with this name runs, new asks go to node-build-agent and wait in its stream until some machine
+// holds the seat. So: let the queued builds finish; roll this controller; register and assign
+// build-agent to the machines that build; unassign builder and forget it and its seat's stream.
 const TheBuildMachine = "node-build-agent"
 
 // BuildWork is where a build request lands, and BuildOutcome is where its result does. Derived from

internal/link/builds_nats_test.go

@@ -245,3 +245,83 @@ func TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue(t *testing.T) {
 func quietLog() *log.Logger { return log.New(io.Discard, "", 0) }
 
 var _ = quietLog
+
+// Two machines holding the role share one queue (novox/hq ADR 0190): three asks, each machine takes
+// one and the third waits until one of them is done; an ask is never handed to a machine that is
+// busy; and a machine that stops mid-ask leaves its ask to the other.
+func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+
+	for _, id := range []string{"w-1", "w-2", "w-3"} {
+		body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"})
+		if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	type taken struct{ machine, id string }
+	took := make(chan taken, 8)
+	release := map[string]chan struct{}{"anchor": make(chan struct{}), "laptop": make(chan struct{})}
+	machines := map[string]BuildMachine{}
+	for _, name := range []string{"anchor", "laptop"} {
+		name := name
+		m := MachineOverNATS(js, name)
+		machines[name] = m
+		defer m.Close()
+		go func() {
+			_ = m.Take(ctx, func(ctx context.Context, work Build) {
+				took <- taken{name, work.Request().ID}
+				<-release[name]
+				_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: name})
+				_ = work.Done()
+			})
+		}()
+	}
+
+	// Each machine took exactly one, and they are different asks.
+	first := map[string]string{}
+	for i := 0; i < 2; i++ {
+		select {
+		case got := <-took:
+			if _, twice := first[got.machine]; twice {
+				t.Fatalf("%s was handed a second ask while busy with its first", got.machine)
+			}
+			first[got.machine] = got.id
+		case <-time.After(10 * time.Second):
+			t.Fatalf("only %d machine(s) took work; two idle holders should both have", len(first))
+		}
+	}
+	if first["anchor"] == first["laptop"] {
+		t.Fatalf("both machines took %q: the queue is not shared, it is copied", first["anchor"])
+	}
+	// The third waits: nobody is free.
+	select {
+	case got := <-took:
+		t.Fatalf("%s was handed %s while both machines were busy", got.machine, got.id)
+	case <-time.After(2 * time.Second):
+	}
+	// One finishes, and only then is the third taken — by that machine, the one that is free.
+	close(release["anchor"])
+	release["anchor"] = make(chan struct{})
+	select {
+	case got := <-took:
+		if got.machine != "anchor" {
+			t.Fatalf("the third ask went to %s, which is still busy", got.machine)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("the third ask was never taken after a machine became free")
+	}
+	// A machine that stops mid-ask leaves its ask unacknowledged, and the ack wait brings it round
+	// to whoever is left — the path TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue proves with
+	// an explicit hand-back, because the real wait is a minute. Here: the laptop goes, anchor
+	// finishes, and with nothing queued nothing more is taken by the machine that is left.
+	machines["laptop"].Close()
+	close(release["anchor"])
+	select {
+	case got := <-took:
+		t.Fatalf("%s took %s; the queue should be empty", got.machine, got.id)
+	case <-time.After(2 * time.Second):
+	}
+}