From 905f3363c9eddd9bd6b76df2ede268f9c677c375 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 2 Oct 2026 22:35:39 +0200 Subject: [PATCH] Two machines holding the build role share one queue, and neither is handed an ask while busy (hq ADR 0190) Against a real bus: three asks, two machines; each takes one, the third waits until one is free and then goes to that one; a machine that stops leaves nothing taken twice. The redelivery of an ask a dead machine held is the ack wait's, proven by the hand-back test beside this one. And the order a live mesh switches over in, written where the role is named: queued builds first, then this controller, then build-agent assigned where machines build, then the builder and the old seat's stream forgotten. --- internal/link/builds.go | 7 +++ internal/link/builds_nats_test.go | 80 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/internal/link/builds.go b/internal/link/builds.go index d71c3be..2f0b599 100644 --- a/internal/link/builds.go +++ b/internal/link/builds.go @@ -22,6 +22,13 @@ import ( // TheBuildMachine is the role a build is submitted to: node-scoped, held on every machine that // builds, and the work shared among them (novox/hq ADR 0190). The name stays for every caller; what // it names moved from the mesh's one build machine to whichever build agent is idle. +// +// **Switching a live mesh over, in order.** The old seat's stream and worker +// (SEAT_MESH_BUILD_MACHINE, SEAT_MESH_BUILD_MACHINE_worker) stay on the bus until removed by hand, +// and the builder module keeps draining them while it is assigned. From the moment a controller +// with this name runs, new asks go to node-build-agent and wait in its stream until some machine +// holds the seat. So: let the queued builds finish; roll this controller; register and assign +// build-agent to the machines that build; unassign builder and forget it and its seat's stream. const TheBuildMachine = "node-build-agent" // BuildWork is where a build request lands, and BuildOutcome is where its result does. Derived from diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index 38971a1..9bac7bd 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -245,3 +245,83 @@ func TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue(t *testing.T) { func quietLog() *log.Logger { return log.New(io.Discard, "", 0) } var _ = quietLog + +// Two machines holding the role share one queue (novox/hq ADR 0190): three asks, each machine takes +// one and the third waits until one of them is done; an ask is never handed to a machine that is +// busy; and a machine that stops mid-ask leaves its ask to the other. +func TestNatsTwoMachinesShareTheWorkAndNeitherIsHandedMoreThanItCanTake(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + for _, id := range []string{"w-1", "w-2", "w-3"} { + body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + } + + type taken struct{ machine, id string } + took := make(chan taken, 8) + release := map[string]chan struct{}{"anchor": make(chan struct{}), "laptop": make(chan struct{})} + machines := map[string]BuildMachine{} + for _, name := range []string{"anchor", "laptop"} { + name := name + m := MachineOverNATS(js, name) + machines[name] = m + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + took <- taken{name, work.Request().ID} + <-release[name] + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: name}) + _ = work.Done() + }) + }() + } + + // Each machine took exactly one, and they are different asks. + first := map[string]string{} + for i := 0; i < 2; i++ { + select { + case got := <-took: + if _, twice := first[got.machine]; twice { + t.Fatalf("%s was handed a second ask while busy with its first", got.machine) + } + first[got.machine] = got.id + case <-time.After(10 * time.Second): + t.Fatalf("only %d machine(s) took work; two idle holders should both have", len(first)) + } + } + if first["anchor"] == first["laptop"] { + t.Fatalf("both machines took %q: the queue is not shared, it is copied", first["anchor"]) + } + // The third waits: nobody is free. + select { + case got := <-took: + t.Fatalf("%s was handed %s while both machines were busy", got.machine, got.id) + case <-time.After(2 * time.Second): + } + // One finishes, and only then is the third taken — by that machine, the one that is free. + close(release["anchor"]) + release["anchor"] = make(chan struct{}) + select { + case got := <-took: + if got.machine != "anchor" { + t.Fatalf("the third ask went to %s, which is still busy", got.machine) + } + case <-time.After(10 * time.Second): + t.Fatal("the third ask was never taken after a machine became free") + } + // A machine that stops mid-ask leaves its ask unacknowledged, and the ack wait brings it round + // to whoever is left — the path TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue proves with + // an explicit hand-back, because the real wait is a minute. Here: the laptop goes, anchor + // finishes, and with nothing queued nothing more is taken by the machine that is left. + machines["laptop"].Close() + close(release["anchor"]) + select { + case got := <-took: + t.Fatalf("%s took %s; the queue should be empty", got.machine, got.id) + case <-time.After(2 * time.Second): + } +}