Let a build agent be paused, have a build killed, and end an ask cancelled as it took it (hq ADR 0219)

A queued ask could only be waited out and a running build only ended by
stopping the machine, which redelivered it elsewhere. The holder now serves
current, kill, pause and resume on its own machine's subjects; a kill ends
the build's process group and labelled containers and settles the ask as
failed, killed by hand; pause is kept in the workspace across a restart and
said on the bus. The controller writes cancelled ids to a cancelled set the
holder reads on taking an ask, closing the race a delete alone leaves.

cmd/mesh-builder/main.go

@@ -19,11 +19,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/url"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/builder"
@@ -107,48 +109,89 @@ func run() error {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	machine, err := takeWorkFrom(credential, on)
+	js, seat, err := dialFor(credential)
 	if err != nil {
 		return err
 	}
+	defer js.Close()
+
+	// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
+	// paused flag is read from the workspace before anything is taken, so a holder restarted while
+	// paused takes nothing.
+	h := newHolder(on, seat, workspace, func(state link.HolderState) error {
+		body, err := json.Marshal(state)
+		if err != nil {
+			return err
+		}
+		_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
+		return err
+	})
+	if h.Paused() {
+		fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
+	}
+	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
 	defer machine.Close()
 
+	// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
+	// one serves none, and a subscription its holder has no grant for would be refused for ever.
+	if seat == link.TheBuildMachine {
+		stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
+			log.New(os.Stderr, "", 0))
+		if err != nil {
+			return err
+		}
+		defer stopServing()
+		go h.stateWhilePaused(ctx, time.Hour)
+	}
+
 	fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
 	publisher := builder.Registry{Address: registry, Run: builder.Command}
 
 	return machine.Take(ctx, func(ctx context.Context, work link.Build) {
-		answer(ctx, publisher, on, workspace, work)
+		answer(ctx, publisher, on, workspace, work, h)
 	})
 }
 
-// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
+// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
+// holds.
 //
 // **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
 // machine told about both would take work from one and answer on the other, and every log line would
 // say it was fine.
-func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
+func dialFor(credential Credential) (*broker.JetStream, string, error) {
 	// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
 	// A credential for the mesh's bus carries user, password and fingerprint beside the address,
 	// and that is enough to dial it, pinned.
 	if !credential.onTheNewBus() {
-		return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
+		return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
 	}
 	js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
 	// handover): the mesh issues a build machine's credential naming the seat its module claims,
 	// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
 	seat := link.BuildSeatClaimed(credential.seatsClaimed())
 	fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
-	return link.MachineOverNATSOn(js, on, seat), nil
+	return js, seat, nil
 }
 
 // answer does one build and says what happened, whichever way it went.
-func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
+func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
 	request := work.Request()
 
+	// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
+	// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
+	// reaches it through the parent, and that keeps today's meaning below.
+	building, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var mine *running
+	if h != nil {
+		mine = h.begin(request, cancel)
+		defer h.end(mine)
+	}
+
 	// **First thing, and to stdout.** A build request that arrives and produces no visible line until
 	// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
 	// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
@@ -162,6 +205,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 	say := func(step, message string) {
 		fmt.Fprintf(os.Stderr, "  [%s] %s\n", step, message)
 		work.Say(step, message)
+		if mine != nil && step != "run" && step != "output" {
+			h.stepped(mine, step)
+		}
 	}
 	builder.Said = say
 	defer func() { builder.Said = nil }()
@@ -188,11 +234,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		// The package-registry credential is a build input, so it is resolved before the clone: a
 		// build that could not have resolved its dependencies is refused in front of the reason, not
 		// after a clone that then fails at npm ci.
-		built, err = builder.Build(ctx, builder.Command, publisher,
+		// Every container it starts is labelled with its id, so a kill finds what outlived the
+		// docker client (ADR 0219).
+		built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
 			request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
 			forgeFrom(), say, request.Seats)
 	}
-	if err != nil {
+	// Only a build the kill ended: one that finished in the moment the kill arrived built, and says so.
+	killed := err != nil && mine != nil && h.wasKilled(mine)
+	if killed {
+		// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
+		// error it ended with is the kill's consequence, not a fault of the source.
+		result.Failed = link.KilledByHand
+		say("failed", link.KilledByHand)
+	} else if err != nil {
 		// A failure is a result. A build that fails and says nothing is indistinguishable from a
 		// builder that is not running, and those want completely different responses.
 		result.Failed = err.Error()
@@ -217,7 +272,17 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		}
 	}
 
-	if err := work.Announce(ctx, result); err != nil {
+	// A killed build is announced on a context of its own: the build's was the one cancelled, and the
+	// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
+	// to be built again. A machine being stopped is the other case and keeps its meaning: the
+	// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
+	announcing := ctx
+	if killed {
+		fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
+		defer stop()
+		announcing = fresh
+	}
+	if err := work.Announce(announcing, result); err != nil {
 		// Said, not fatal: the build happened. A build reported as failed because announcing it
 		// failed is a lie about work that was done — and the request stays unsettled below only if
 		// nothing was said at all, so another machine can try.