Let a build agent be paused, have a build killed, and end an ask cancelled as it took it (hq ADR 0219)

A queued ask could only be waited out and a running build only ended by
stopping the machine, which redelivered it elsewhere. The holder now serves
current, kill, pause and resume on its own machine's subjects; a kill ends
the build's process group and labelled containers and settles the ask as
failed, killed by hand; pause is kept in the workspace across a restart and
said on the bus. The controller writes cancelled ids to a cancelled set the
holder reads on taking an ask, closing the race a delete alone leaves.

internal/builder/kill_test.go

@@ -0,0 +1,103 @@
+package builder
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills
+// the command's whole process group, not the one process the context knows about.
+func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) {
+	dir := t.TempDir()
+	child := filepath.Join(dir, "child")
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	began := time.Now()
+	go func() {
+		// A shell that starts a grandchild and waits on it: killing the shell alone would leave the
+		// grandchild running, holding the output open.
+		_, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`)
+		done <- err
+	}()
+	var pid int
+	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
+		if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" {
+			pid, _ = strconv.Atoi(strings.TrimSpace(string(raw)))
+			break
+		}
+	}
+	if pid == 0 {
+		t.Fatal("the command never started its child")
+	}
+	cancel()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("a killed command reported success")
+		}
+	case <-time.After(KillWait + 5*time.Second):
+		t.Fatal("the killed command never returned")
+	}
+	if took := time.Since(began); took > KillWait {
+		t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took)
+	}
+	for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) {
+		if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) {
+			return
+		}
+	}
+	_ = syscall.Kill(pid, syscall.SIGKILL)
+	t.Fatalf("the grandchild %d outlived the kill", pid)
+}
+
+// Every `docker run` a build starts carries its id as a label; nothing else is touched.
+func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) {
+	got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
+	if want := []string{"run", "--label", "mesh.build=build-1", "--rm", "img", "sh"}; !reflect.DeepEqual(got, want) {
+		t.Errorf("docker run became %v", got)
+	}
+	for _, c := range []struct {
+		name string
+		args []string
+	}{{"docker", []string{"build", "."}}, {"git", []string{"run"}}, {"docker", nil}} {
+		if got := LabelledArgs(c.name, c.args, "build-1"); !reflect.DeepEqual(got, c.args) {
+			t.Errorf("%s %v became %v", c.name, c.args, got)
+		}
+	}
+	var ran [][]string
+	run := Labelled(func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, append([]string{name}, args...))
+		return "", nil
+	}, "build-2")
+	_, _ = run(context.Background(), "", "docker", "run", "img")
+	if want := [][]string{{"docker", "run", "--label", "mesh.build=build-2", "img"}}; !reflect.DeepEqual(ran, want) {
+		t.Errorf("ran %v", ran)
+	}
+}
+
+// What a kill removes is found by the label, and only what it finds.
+func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
+	var ran []string
+	run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		if args[0] == "ps" {
+			return "c1\nc2\n", nil
+		}
+		return "", nil
+	}
+	n, err := RemoveContainersOf(context.Background(), run, "build-3")
+	if err != nil || n != 2 {
+		t.Fatalf("%d %v", n, err)
+	}
+	if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
+		t.Errorf("ran %v", ran)
+	}
+}