Let a build agent be paused, have a build killed, and end an ask cancelled as it took it (hq ADR 0219)

A queued ask could only be waited out and a running build only ended by
stopping the machine, which redelivered it elsewhere. The holder now serves
current, kill, pause and resume on its own machine's subjects; a kill ends
the build's process group and labelled containers and settles the ask as
failed, killed by hand; pause is kept in the workspace across a restart and
said on the bus. The controller writes cancelled ids to a cancelled set the
holder reads on taking an ask, closing the race a delete alone leaves.

cmd/mesh-builder/holder.go

@@ -0,0 +1,295 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/builder"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
+//
+// **The queue is the controller's; the build running here is this machine's.** The controller can
+// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
+// on this machine and containers in this machine's runtime, and only this machine can end them. So
+// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
+// it, pause, resume.
+//
+// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
+// set per build — so "the build running here" is one or none, and kill names it by id so a call
+// that arrives as one build ends and the next begins cannot end the wrong one.
+
+// holder is this machine's state as a holder of the build seat.
+type holder struct {
+	on, seat string
+	// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
+	// rather than silently taking work again.
+	workspace string
+	// say publishes this machine's state: whether it takes work (link.HolderState).
+	say func(link.HolderState) error
+	// remove runs what a kill needs outside the build: the containers left behind.
+	remove builder.Runner
+
+	mu      sync.Mutex
+	paused  bool
+	running *running
+}
+
+// running is the build this machine is doing.
+type running struct {
+	request link.BuildRequest
+	step    string
+	started time.Time
+	cancel  context.CancelFunc
+	killed  bool
+	done    chan struct{}
+}
+
+// pausedFile is where the flag lives in the workspace.
+func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
+
+// newHolder reads the paused flag the workspace keeps.
+func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
+	h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
+	if _, err := os.Stat(pausedFile(workspace)); err == nil {
+		h.paused = true
+	}
+	return h
+}
+
+// Paused is asked by the taking loop before every fetch.
+func (h *holder) Paused() bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.paused
+}
+
+// setPaused records the flag in the workspace first and then in memory, so what this holder says
+// it is and what it would be after a restart never differ.
+func (h *holder) setPaused(paused bool) error {
+	path := pausedFile(h.workspace)
+	if paused {
+		if err := os.MkdirAll(h.workspace, 0o755); err != nil {
+			return err
+		}
+		if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+			return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
+		}
+	} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
+	}
+	h.mu.Lock()
+	h.paused = paused
+	h.mu.Unlock()
+	h.announce()
+	return nil
+}
+
+// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
+// controller reading an older state is a plan read as late rather than a build lost.
+func (h *holder) announce() {
+	if h.say == nil {
+		return
+	}
+	if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
+		fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
+	}
+}
+
+// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
+// a pause outlives the events stream's retention.
+func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
+	h.announce()
+	tick := time.NewTicker(every)
+	defer tick.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+			if h.Paused() {
+				h.announce()
+			}
+		}
+	}
+}
+
+// begin records the build this machine took, with the cancel that ends it.
+func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
+	r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
+	h.mu.Lock()
+	h.running = r
+	h.mu.Unlock()
+	return r
+}
+
+// end clears it, and lets a kill waiting on it know it is over.
+func (h *holder) end(r *running) {
+	h.mu.Lock()
+	if h.running == r {
+		h.running = nil
+	}
+	h.mu.Unlock()
+	close(r.done)
+}
+
+// stepped records the step a build is at, for `current`.
+func (h *holder) stepped(r *running, step string) {
+	h.mu.Lock()
+	r.step = step
+	h.mu.Unlock()
+}
+
+// wasKilled is whether a person killed this build.
+func (h *holder) wasKilled(r *running) bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return r.killed
+}
+
+// currentBuild is what `current` answers.
+type currentBuild struct {
+	On      string `json:"on"`
+	Paused  bool   `json:"paused"`
+	Running *struct {
+		ID         string `json:"id"`
+		Repository string `json:"repository"`
+		Path       string `json:"path,omitempty"`
+		Ref        string `json:"ref,omitempty"`
+		Step       string `json:"step,omitempty"`
+		Started    string `json:"started"`
+		Elapsed    string `json:"elapsed"`
+	} `json:"running,omitempty"`
+	Said string `json:"said"`
+}
+
+func (h *holder) current() currentBuild {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	out := currentBuild{On: h.on, Paused: h.paused}
+	taking := "taking builds"
+	if h.paused {
+		taking = "paused, taking no new build"
+	}
+	if h.running == nil {
+		out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
+		return out
+	}
+	r := h.running
+	elapsed := time.Since(r.started).Round(time.Second)
+	out.Running = &struct {
+		ID         string `json:"id"`
+		Repository string `json:"repository"`
+		Path       string `json:"path,omitempty"`
+		Ref        string `json:"ref,omitempty"`
+		Step       string `json:"step,omitempty"`
+		Started    string `json:"started"`
+		Elapsed    string `json:"elapsed"`
+	}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
+		r.started.UTC().Format(time.RFC3339), elapsed.String()}
+	out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
+		h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
+	return out
+}
+
+// kill ends the build with this id, if it is the one running here: its context cancelled — which
+// kills each command's process group — and the containers it started removed by their label. The
+// build's own goroutine then announces it failed, killed by hand, and settles the ask, so it is not
+// redelivered; this waits a while for that, to say it happened.
+func (h *holder) kill(id string) (string, error) {
+	h.mu.Lock()
+	r := h.running
+	if r == nil || r.request.ID != id {
+		doing := "nothing"
+		if r != nil {
+			doing = r.request.ID
+		}
+		h.mu.Unlock()
+		return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
+	}
+	r.killed = true
+	cancel, done := r.cancel, r.done
+	h.mu.Unlock()
+
+	cancel()
+	cleanup, stop := context.WithTimeout(context.Background(), 30*time.Second)
+	defer stop()
+	removed, err := builder.RemoveContainersOf(cleanup, h.remove, id)
+	containers := fmt.Sprintf("%d container(s) it started removed", removed)
+	if err != nil {
+		containers = "its containers could not be listed or removed: " + err.Error()
+	}
+	select {
+	case <-done:
+		return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as "+
+			"failed, %s — settled, so it is not handed to another machine", id, r.request.Repository, h.on,
+			containers, link.KilledByHand), nil
+	case <-time.After(30 * time.Second):
+		return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
+			"its outcome is in", id, h.on, containers), nil
+	}
+}
+
+// handlers are the seat's verbs, as this machine answers them.
+func (h *holder) handlers() map[string]link.ToolHandler {
+	return map[string]link.ToolHandler{
+		"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
+		"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
+			var args struct {
+				ID string `json:"id"`
+			}
+			if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
+				return nil, errors.New("kill needs the build's id")
+			}
+			said, err := h.kill(strings.TrimSpace(args.ID))
+			if err != nil {
+				return nil, err
+			}
+			return map[string]any{"said": said}, nil
+		},
+		"pause": func(context.Context, json.RawMessage) (any, error) {
+			if err := h.setPaused(true); err != nil {
+				return nil, err
+			}
+			said := h.on + " is paused: it takes no new build until resumed"
+			if c := h.current(); c.Running != nil {
+				said += "; " + c.Running.ID + " runs on and finishes"
+			}
+			return map[string]any{"said": said, "paused": true}, nil
+		},
+		"resume": func(context.Context, json.RawMessage) (any, error) {
+			if err := h.setPaused(false); err != nil {
+				return nil, err
+			}
+			return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
+		},
+	}
+}
+
+func orNothing(s string) string {
+	if s == "" {
+		return "its start"
+	}
+	return s
+}
+
+// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
+// log it would be is the one being ended.
+func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Dir = dir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
+	}
+	return string(out), nil
+}

cmd/mesh-builder/holder_test.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
+// 0219), and what it says about itself follows.
+func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
+	workspace := t.TempDir()
+	var said []link.HolderState
+	h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
+		said = append(said, s)
+		return nil
+	})
+	if h.Paused() {
+		t.Fatal("a new holder starts paused")
+	}
+	if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
+		t.Fatalf("paused: %v, said %+v", h.Paused(), said)
+	}
+	again := newHolder("ace", link.TheBuildMachine, workspace, nil)
+	if !again.Paused() {
+		t.Fatal("restarted, the holder forgot it was paused")
+	}
+	if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
+		t.Fatal("resumed, the holder came back paused")
+	}
+}
+
+// kill ends the build with that id — its context, which ends its commands — removes what it left by
+// label, and refuses an id it is not building.
+func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
+	h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
+	var removed []string
+	h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		removed = append(removed, name+" "+strings.Join(args, " "))
+		if args[0] == "ps" {
+			return "c1\n", nil
+		}
+		return "", nil
+	}
+	kill := h.handlers()["kill"]
+	if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
+		t.Fatal("killed a build while none ran")
+	}
+
+	building, cancel := context.WithCancel(context.Background())
+	r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
+	h.stepped(r, "image")
+	if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
+		t.Fatalf("current says %+v", c)
+	}
+	if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
+		!strings.Contains(err.Error(), "build-1") {
+		t.Fatalf("killed another id: %v", err)
+	}
+	// The build's own goroutine: it ends when its context does, as a build's commands do.
+	go func() {
+		<-building.Done()
+		h.end(r)
+	}()
+	answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if building.Err() == nil {
+		t.Fatal("the build's context was not cancelled")
+	}
+	if !h.wasKilled(r) {
+		t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
+	}
+	said := answer.(map[string]any)["said"].(string)
+	if !strings.Contains(said, "1 container(s)") || !strings.Contains(said, link.KilledByHand) {
+		t.Errorf("kill said %q", said)
+	}
+	if len(removed) != 2 || !strings.Contains(removed[0], "label=mesh.build=build-1") {
+		t.Errorf("removed %v", removed)
+	}
+	if c := h.current(); c.Running != nil {
+		t.Errorf("after the kill current says %+v", c)
+	}
+}

cmd/mesh-builder/main.go

@@ -19,11 +19,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/url"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/builder"
@@ -107,48 +109,89 @@ func run() error {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	machine, err := takeWorkFrom(credential, on)
+	js, seat, err := dialFor(credential)
 	if err != nil {
 		return err
 	}
+	defer js.Close()
+
+	// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
+	// paused flag is read from the workspace before anything is taken, so a holder restarted while
+	// paused takes nothing.
+	h := newHolder(on, seat, workspace, func(state link.HolderState) error {
+		body, err := json.Marshal(state)
+		if err != nil {
+			return err
+		}
+		_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
+		return err
+	})
+	if h.Paused() {
+		fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
+	}
+	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
 	defer machine.Close()
 
+	// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
+	// one serves none, and a subscription its holder has no grant for would be refused for ever.
+	if seat == link.TheBuildMachine {
+		stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
+			log.New(os.Stderr, "", 0))
+		if err != nil {
+			return err
+		}
+		defer stopServing()
+		go h.stateWhilePaused(ctx, time.Hour)
+	}
+
 	fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
 	publisher := builder.Registry{Address: registry, Run: builder.Command}
 
 	return machine.Take(ctx, func(ctx context.Context, work link.Build) {
-		answer(ctx, publisher, on, workspace, work)
+		answer(ctx, publisher, on, workspace, work, h)
 	})
 }
 
-// takeWorkFrom opens this machine's link to whichever bus the mesh is on.
+// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
+// holds.
 //
 // **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
 // machine told about both would take work from one and answer on the other, and every log line would
 // say it was fine.
-func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) {
+func dialFor(credential Credential) (*broker.JetStream, string, error) {
 	// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
 	// A credential for the mesh's bus carries user, password and fingerprint beside the address,
 	// and that is enough to dial it, pinned.
 	if !credential.onTheNewBus() {
-		return nil, fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
+		return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
 	}
 	js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
 	// handover): the mesh issues a build machine's credential naming the seat its module claims,
 	// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
 	seat := link.BuildSeatClaimed(credential.seatsClaimed())
 	fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
-	return link.MachineOverNATSOn(js, on, seat), nil
+	return js, seat, nil
 }
 
 // answer does one build and says what happened, whichever way it went.
-func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) {
+func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
 	request := work.Request()
 
+	// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
+	// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
+	// reaches it through the parent, and that keeps today's meaning below.
+	building, cancel := context.WithCancel(ctx)
+	defer cancel()
+	var mine *running
+	if h != nil {
+		mine = h.begin(request, cancel)
+		defer h.end(mine)
+	}
+
 	// **First thing, and to stdout.** A build request that arrives and produces no visible line until
 	// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
 	// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
@@ -162,6 +205,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 	say := func(step, message string) {
 		fmt.Fprintf(os.Stderr, "  [%s] %s\n", step, message)
 		work.Say(step, message)
+		if mine != nil && step != "run" && step != "output" {
+			h.stepped(mine, step)
+		}
 	}
 	builder.Said = say
 	defer func() { builder.Said = nil }()
@@ -188,11 +234,20 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		// The package-registry credential is a build input, so it is resolved before the clone: a
 		// build that could not have resolved its dependencies is refused in front of the reason, not
 		// after a clone that then fails at npm ci.
-		built, err = builder.Build(ctx, builder.Command, publisher,
+		// Every container it starts is labelled with its id, so a kill finds what outlived the
+		// docker client (ADR 0219).
+		built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
 			request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
 			forgeFrom(), say, request.Seats)
 	}
-	if err != nil {
+	// Only a build the kill ended: one that finished in the moment the kill arrived built, and says so.
+	killed := err != nil && mine != nil && h.wasKilled(mine)
+	if killed {
+		// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
+		// error it ended with is the kill's consequence, not a fault of the source.
+		result.Failed = link.KilledByHand
+		say("failed", link.KilledByHand)
+	} else if err != nil {
 		// A failure is a result. A build that fails and says nothing is indistinguishable from a
 		// builder that is not running, and those want completely different responses.
 		result.Failed = err.Error()
@@ -217,7 +272,17 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
 		}
 	}
 
-	if err := work.Announce(ctx, result); err != nil {
+	// A killed build is announced on a context of its own: the build's was the one cancelled, and the
+	// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
+	// to be built again. A machine being stopped is the other case and keeps its meaning: the
+	// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
+	announcing := ctx
+	if killed {
+		fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
+		defer stop()
+		announcing = fresh
+	}
+	if err := work.Announce(announcing, result); err != nil {
 		// Said, not fatal: the build happened. A build reported as failed because announcing it
 		// failed is a lie about work that was done — and the request stays unsettled below only if
 		// nothing was said at all, so another machine can try.

cmd/mesh-controller/push.go

@@ -1142,6 +1142,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
 	if err := broker.RaiseSeats(js, inventory.MeshSeats(), holders); err != nil {
 		return err
 	}
+	// And each work queue's cancelled set (novox/hq ADR 0219), so a holder taking an ask can ask
+	// whether it was cancelled the moment it took it.
+	if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
+		return err
+	}
 	// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
 	// registration, so a module reading one may watch it before its owner runs anywhere. One that
 	// nothing declares any more is said and kept — what it holds is data.