2026-10-07 16:28:44 +00:00
3 changed files with 624 additions and 4 deletions
@@ -0,0 +1,410 @@
+package replays
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// The bed (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): **a declaration is proved before it is trusted.**
+//
+// Two of the nineteen image checks the catalogue's containers ship read *unhealthy* while working in the
+// mesh's hands — the studio's asked an address its program does not bind, the flow editor's read settings
+// from a path the module mounted elsewhere. Read without proof, they would have put back two good builds.
+// So the catalogue's merge check starts every resource whose declared check or image the change touches,
+// **alone**, on a throwaway runtime, and requires its check to see the program within its grace.
+//
+// **What it proves is the check's wiring**: that it can see the program working — a property of the image
+// and the declaration, not of the mesh. A resource is started with what the module declares and nothing of
+// the mesh's: its literal environment, its arguments, its files where the module writes them in full, and an
+// empty directory for every other place it mounts; no secret, no binding, no provider. So:
+//
+//   - a check naming a provision in `needs` gets no provider here, and must only reach the program — an
+//     http answer of any status, a connect;
+//   - a program that does not stay up alone — it needs what the mesh gives it — is said as not proved here,
+//     and judged on the first machine's gate, never passed as proved and never failed;
+//   - a program that stays up while its check does not see it **fails**: that is the studio's fault.
+//
+// A tool check and a unit's own readiness need the mesh and a machine; they are said, not proved.
+
+// BedResource is one resource the bed proves: its module, its id, and the resource as the manifest says it.
+type BedResource struct {
+	Module   string
+	ID       string
+	Resource map[string]any
+	// Listens are the module's endpoints, by name: the container's own port for each.
+	Listens map[string]int
+}
+
+// BedVerdict is what the bed found of one resource.
+type BedVerdict struct {
+	Proved bool
+	// Failed says the program stayed up and its check did not see it: the change's fault.
+	Failed bool
+	Said   string
+}
+
+// BedLook is how often the bed looks: often, because it proves the wiring, not the timing.
+var BedLook = 2 * time.Second
+
+// BedFloor is the least the bed waits for a check to pass, whatever the grace: a program needs a moment to
+// listen even when the module declared no grace.
+var BedFloor = 30 * time.Second
+
+// ChangedHealth is every long-running container resource of the catalogue at root that declares `health`
+// and whose resource changed against base (a git ref in that checkout): its declaration, its image, or
+// anything else of it. A manifest new on this branch counts whole. Read through git; an error when the
+// base cannot be read.
+func ChangedHealth(root, base string, show func(ref, path string) ([]byte, error)) ([]BedResource, error) {
+	paths, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
+	if err != nil {
+		return nil, err
+	}
+	var out []BedResource
+	for _, p := range paths {
+		raw, err := os.ReadFile(p)
+		if err != nil {
+			return nil, err
+		}
+		rel, _ := filepath.Rel(root, p)
+		was := map[string]string{}
+		if before, err := show(base, rel); err == nil {
+			for _, r := range resourcesOf(before) {
+				was[fmt.Sprint(r["id"])] = canonical(r)
+			}
+		}
+		module, listens, resources := manifestParts(raw)
+		for _, r := range resources {
+			if _, declared := r["health"]; !declared || fmt.Sprint(r["type"]) != "container" || !stays(r) {
+				continue
+			}
+			if was[fmt.Sprint(r["id"])] == canonical(r) {
+				continue
+			}
+			out = append(out, BedResource{Module: module, ID: fmt.Sprint(r["id"]), Resource: r, Listens: listens})
+		}
+	}
+	sort.Slice(out, func(a, b int) bool { return out[a].Module+"."+out[a].ID < out[b].Module+"."+out[b].ID })
+	return out, nil
+}
+
+func stays(r map[string]any) bool {
+	once, _ := r["run-once"].(bool)
+	return !once && r["schedule"] == nil
+}
+
+func canonical(r map[string]any) string {
+	raw, _ := json.Marshal(r)
+	return string(raw)
+}
+
+func resourcesOf(raw []byte) []map[string]any {
+	_, _, rs := manifestParts(raw)
+	return rs
+}
+
+func manifestParts(raw []byte) (string, map[string]int, []map[string]any) {
+	var m struct {
+		Module  string `json:"module"`
+		Listens []struct {
+			Name string `json:"name"`
+			Port int    `json:"port"`
+		} `json:"listens"`
+		Resources []map[string]any `json:"resources"`
+	}
+	_ = json.Unmarshal(raw, &m)
+	listens := map[string]int{}
+	for _, l := range m.Listens {
+		if l.Name != "" {
+			listens[l.Name] = l.Port
+		}
+	}
+	return m.Module, listens, m.Resources
+}
+
+// placeholder is anything the mesh fills in on a machine: a value holding one is not the module's alone.
+var placeholder = regexp.MustCompile(`\$\{[^}]*\}`)
+
+// Prove starts one resource alone on the runtime and looks at it with its declared check until the check
+// sees it, the program stops, or its grace (at least BedFloor) runs out. Everything it made is removed.
+func (d *Docker) Prove(ctx context.Context, r BedResource, files map[string]string) BedVerdict {
+	h, _ := r.Resource["health"].(map[string]any)
+	kind := fmt.Sprint(h["kind"])
+	switch kind {
+	case "tool", "unit":
+		return BedVerdict{Said: fmt.Sprintf("%s.%s: a %s check needs the mesh and a machine; judged on the first "+
+			"machine's gate", r.Module, r.ID, kind)}
+	}
+	image, _ := r.Resource["image"].(string)
+	if image == "" || placeholder.MatchString(image) {
+		return BedVerdict{Said: fmt.Sprintf("%s.%s: its image is built by the mesh (%v); judged on the first machine's gate",
+			r.Module, r.ID, r.Resource["artifact"])}
+	}
+	if err := d.Pull(ctx, image); err != nil {
+		return BedVerdict{Said: fmt.Sprintf("%s.%s: its image could not be fetched here: %v", r.Module, r.ID, oneLine(err.Error()))}
+	}
+	scratch, err := os.MkdirTemp("", "mesh-bed-")
+	if err != nil {
+		return BedVerdict{Said: err.Error()}
+	}
+	defer os.RemoveAll(scratch)
+	// Its literal environment, and every env-file the module writes in full.
+	var env []string
+	if e, ok := r.Resource["env"].(map[string]any); ok {
+		for k, v := range e {
+			if s := fmt.Sprint(v); !placeholder.MatchString(s) {
+				env = append(env, k+"="+s)
+			}
+		}
+	}
+	if list, ok := r.Resource["env-file"].([]any); ok {
+		for _, f := range list {
+			if content, ok := files[fmt.Sprint(f)]; ok {
+				env = append(env, envLines(content)...)
+			}
+		}
+	}
+	sort.Strings(env)
+	// Its mounts: a file the module writes in full, or an empty place of its own.
+	var binds []string
+	if list, ok := r.Resource["volumes"].([]any); ok {
+		for i, v := range list {
+			src, dst, rest := splitMount(fmt.Sprint(v))
+			if dst == "" || !strings.HasPrefix(dst, "/") {
+				continue
+			}
+			local := filepath.Join(scratch, "m"+strconv.Itoa(i))
+			if content, ok := files[src]; ok {
+				if err := os.WriteFile(local, []byte(content), 0o644); err != nil {
+					continue
+				}
+			} else if err := os.MkdirAll(local, 0o777); err != nil {
+				continue
+			}
+			_ = os.Chmod(local, 0o777)
+			binds = append(binds, local+":"+dst+rest)
+		}
+	}
+	var args []string
+	if list, ok := r.Resource["args"].([]any); ok {
+		for _, a := range list {
+			args = append(args, fmt.Sprint(a))
+		}
+	}
+	network := fmt.Sprintf("mesh-bed-%d", time.Now().UnixNano())
+	netID, err := d.Network(ctx, network)
+	if err != nil {
+		return BedVerdict{Said: "the bed's network could not be made: " + err.Error()}
+	}
+	defer d.RemoveNetwork(context.Background(), netID)
+
+	grace, _ := time.ParseDuration(fmt.Sprint(h["grace"]))
+	if _, said := h["grace"]; !said {
+		grace = 60 * time.Second
+	}
+	if grace < BedFloor {
+		grace = BedFloor
+	}
+	config := map[string]any{"Image": image, "Env": env, "Labels": map[string]string{d.Label: "bed"}}
+	if len(args) > 0 {
+		config["Cmd"] = args
+	}
+	switch kind {
+	case "exec":
+		config["Healthcheck"] = map[string]any{"Test": []string{"CMD-SHELL", fmt.Sprint(h["command"])},
+			"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1}
+	case "runtime":
+		// The image's own command, adopted by name: an empty Test keeps it.
+		config["Healthcheck"] = map[string]any{"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1,
+			"Retries": 1}
+	}
+	config["HostConfig"] = map[string]any{"Binds": binds, "NetworkMode": network}
+	name := fmt.Sprintf("mesh-bed-%s-%s-%d", r.Module, r.ID, time.Now().UnixNano())
+	var made struct{ ID string }
+	if err := d.call(ctx, http.MethodPost, "/containers/create?name="+name, config, &made); err != nil {
+		return BedVerdict{Said: fmt.Sprintf("%s.%s could not be made here: %v", r.Module, r.ID, oneLine(err.Error()))}
+	}
+	defer d.Remove(context.Background(), made.ID)
+	if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil {
+		return BedVerdict{Said: fmt.Sprintf("%s.%s could not be started here: %v", r.Module, r.ID, oneLine(err.Error()))}
+	}
+
+	port := r.Listens[fmt.Sprint(h["endpoint"])]
+	needs := fmt.Sprint(h["needs"]) != "" && h["needs"] != nil
+	deadline := time.Now().Add(grace)
+	last := "it was never looked at"
+	for {
+		var seen struct {
+			State struct {
+				Running bool
+				Health  *struct {
+					Status string
+					Log    []struct{ Output string }
+				}
+			}
+			NetworkSettings struct {
+				Networks map[string]struct{ IPAddress string }
+			}
+		}
+		if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil {
+			return BedVerdict{Said: err.Error()}
+		}
+		if !seen.State.Running {
+			return BedVerdict{Said: fmt.Sprintf("%s.%s does not stay up alone — it needs what the mesh gives it — so its "+
+				"check is judged on the first machine's gate: %s", r.Module, r.ID, oneLine(lastLine(d.Logs(ctx, made.ID))))}
+		}
+		ok := false
+		switch kind {
+		case "exec", "runtime":
+			hs := seen.State.Health
+			switch {
+			case hs == nil:
+				last = "the container carries no check: its image ships none to adopt"
+			case hs.Status == "healthy":
+				ok = true
+			default:
+				last = "the runtime says " + hs.Status
+				if n := len(hs.Log); n > 0 {
+					last += ": " + oneLine(hs.Log[n-1].Output)
+				}
+			}
+		case "http", "tcp":
+			address := ""
+			for _, n := range seen.NetworkSettings.Networks {
+				address = n.IPAddress
+			}
+			ok, last = look(ctx, kind, address, port, h, needs)
+		default:
+			return BedVerdict{Said: fmt.Sprintf("%s.%s declares a %s check the bed does not know", r.Module, r.ID, kind)}
+		}
+		if ok {
+			return BedVerdict{Proved: true, Said: fmt.Sprintf("%s.%s: its %s check sees the program", r.Module, r.ID, kind)}
+		}
+		if time.Now().After(deadline) || ctx.Err() != nil {
+			return BedVerdict{Failed: true, Said: fmt.Sprintf("%s.%s stays up and its %s check does not see it within %s: %s",
+				r.Module, r.ID, kind, grace, last)}
+		}
+		select {
+		case <-ctx.Done():
+		case <-time.After(BedLook):
+		}
+	}
+}
+
+// look is one http or tcp look at the program, from outside it, as the node-engine makes it. A check that
+// needs a provider only has to reach the program: any answer, a connect.
+func look(ctx context.Context, kind, address string, port int, h map[string]any, needs bool) (bool, string) {
+	if address == "" || port == 0 {
+		return false, "it has no address on the bed's network yet"
+	}
+	at := net.JoinHostPort(address, strconv.Itoa(port))
+	ctx, cancel := context.WithTimeout(ctx, BedLook)
+	defer cancel()
+	if kind == "tcp" {
+		c, err := (&net.Dialer{}).DialContext(ctx, "tcp", at)
+		if err != nil {
+			return false, oneLine(err.Error())
+		}
+		c.Close()
+		return true, ""
+	}
+	scheme, _ := h["scheme"].(string)
+	if scheme == "" {
+		scheme = "http"
+	}
+	path, _ := h["path"].(string)
+	if path == "" {
+		path = "/"
+	}
+	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+at+path, nil)
+	res, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+		Transport: insecure()}).Do(req)
+	if err != nil {
+		return false, oneLine(err.Error())
+	}
+	res.Body.Close()
+	if needs {
+		return true, ""
+	}
+	want, _ := h["status"].(float64)
+	switch {
+	case want != 0 && res.StatusCode != int(want):
+		return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, int(want))
+	case want == 0 && res.StatusCode >= 400:
+		return false, fmt.Sprintf("answered %d", res.StatusCode)
+	}
+	return true, ""
+}
+
+func splitMount(v string) (src, dst, rest string) {
+	parts := strings.SplitN(v, ":", 3)
+	if len(parts) < 2 {
+		return "", "", ""
+	}
+	if len(parts) == 3 {
+		rest = ":" + parts[2]
+	}
+	return parts[0], parts[1], rest
+}
+
+func envLines(content string) []string {
+	var out []string
+	s := bufio.NewScanner(strings.NewReader(content))
+	for s.Scan() {
+		line := strings.TrimSpace(s.Text())
+		if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") || placeholder.MatchString(line) {
+			continue
+		}
+		out = append(out, line)
+	}
+	return out
+}
+
+func lastLine(s string) string {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	return lines[len(lines)-1]
+}
+
+// FilesOf is every file a module writes in full — its content declared, nothing the mesh fills in — by the
+// path it writes it at, as the module names it: what the bed gives a container that mounts or reads it.
+func FilesOf(manifest []byte) map[string]string {
+	_, _, rs := manifestParts(manifest)
+	out := map[string]string{}
+	for _, r := range rs {
+		if fmt.Sprint(r["type"]) != "file" {
+			continue
+		}
+		path, _ := r["path"].(string)
+		content, ok := r["content"].(string)
+		if path == "" || !ok || placeholder.MatchString(content) {
+			continue
+		}
+		out[path] = content
+	}
+	return out
+}
+
+// insecure is a transport that asks whether a program answers, not whom to trust.
+func insecure() *http.Transport {
+	return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true}
+}
+
+// oneLine is the first line of what was said, short.
+func oneLine(s string) string {
+	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
+	if len(line) > 300 {
+		line = line[:300] + "…"
+	}
+	return line
+}
@@ -0,0 +1,161 @@
+package replays
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// **The catalogue's bed** (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): every long-running resource whose
+// declared `health` or image the change touches is started alone and its check must see the program within
+// its grace. Run by every merge check of the catalogue on the build seat, with the change's catalogue at
+// MESH_REPLAY_CATALOGUE; by hand against the catalogue beside the lab. What changed is against
+// MESH_BED_BASE (origin/main by default) in that checkout.
+func TestBedProvesEveryChangedHealthCheck(t *testing.T) {
+	root := orDefault(os.Getenv("MESH_REPLAY_CATALOGUE"), filepath.Join("..", "..", "mesh-catalog"))
+	if _, err := os.Stat(filepath.Join(root, "modules")); err != nil {
+		t.Skipf("no catalogue at %s (MESH_REPLAY_CATALOGUE names it)", root)
+	}
+	base := orDefault(os.Getenv("MESH_BED_BASE"), "origin/main")
+	show := func(ref, path string) ([]byte, error) {
+		return exec.Command("git", "-c", "safe.directory=*", "-C", root, "show", ref+":"+path).Output()
+	}
+	if _, err := exec.Command("git", "-c", "safe.directory=*", "-C", root, "rev-parse", "--verify", base).Output(); err != nil {
+		// What changed cannot be told, so every declared check is proved: the bed never passes what it did
+		// not look at.
+		t.Logf("%s is not in the catalogue at %s (%v): every declared check is proved", base, root, err)
+	}
+	changed, err := ChangedHealth(root, base, show)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(changed) == 0 {
+		t.Logf("no declared check or image of a long-running resource changed against %s: nothing to prove", base)
+		return
+	}
+	docker, ok := DockerFromEnv()
+	if !ok {
+		t.Fatalf("%d changed check(s) and no container runtime to prove them on", len(changed))
+	}
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Minute)
+	defer cancel()
+	for _, r := range changed {
+		manifest, _ := os.ReadFile(filepath.Join(root, "modules", r.Module, "module.json"))
+		v := docker.Prove(ctx, r, FilesOf(manifest))
+		switch {
+		case v.Failed:
+			t.Errorf("FAILS ON THE BED: %s", v.Said)
+		case v.Proved:
+			t.Logf("proved: %s", v.Said)
+		default:
+			t.Logf("NOT PROVED HERE: %s", v.Said)
+		}
+	}
+}
+
+// **R-studio — a check that asks an address the program does not bind fails the bed, not a machine**
+// (novox/hq ADR 0240 Phase D, done when). The hosted database suite's studio binds the address the runtime
+// puts in HOSTNAME, so it answered only on its network address while its image's own check asked localhost:
+// unhealthy for ever while working, until the module set HOSTNAME=0.0.0.0. Adopted by name without proof, it
+// would have put back a good build.
+//
+// The replay is that image in miniature: a web server that binds $HOSTNAME's address, and an image check that
+// asks localhost. Adopted as the module declared it before the fix, the bed fails it; with the fix, it proves
+// it. The image is built here and removed after, with everything the bed made.
+func TestBedFailsTheStudiosFalseUnhealthy(t *testing.T) {
+	docker, ok := DockerFromEnv()
+	if !ok {
+		t.Skip("no container runtime: the studio is a container")
+	}
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)
+	defer cancel()
+	if err := docker.Pull(ctx, "busybox:1.36"); err != nil {
+		t.Fatal(err)
+	}
+	tag := fmt.Sprintf("mesh-replay-studio:%d", time.Now().UnixNano())
+	if err := docker.Build(ctx, tag, StudioDockerfile); err != nil {
+		t.Fatal(err)
+	}
+	defer docker.RemoveImage(context.Background(), tag)
+	was := BedFloor
+	BedFloor = 15 * time.Second
+	defer func() { BedFloor = was }()
+
+	studio := func(env map[string]any) BedResource {
+		r := map[string]any{"id": "studio", "type": "container", "name": "supabase-studio", "image": tag,
+			"health": map[string]any{"kind": "runtime", "grace": "10s"}}
+		if env != nil {
+			r["env"] = env
+		}
+		return BedResource{Module: "supabase", ID: "studio", Resource: r, Listens: map[string]int{}}
+	}
+	before := docker.Prove(ctx, studio(nil), nil)
+	if !before.Failed || !strings.Contains(before.Said, "does not see it") {
+		t.Fatalf("the studio's false unhealthy was not failed on the bed: %+v", before)
+	}
+	t.Logf("before the fix: %s", before.Said)
+	after := docker.Prove(ctx, studio(map[string]any{"HOSTNAME": "0.0.0.0"}), nil)
+	if !after.Proved {
+		t.Fatalf("the studio with HOSTNAME=0.0.0.0 was not proved: %+v", after)
+	}
+	t.Logf("with the fix: %s", after.Said)
+}
+
+// StudioDockerfile is the studio in miniature: it serves on the address HOSTNAME names, and its own check
+// asks localhost, as the studio's image does.
+const StudioDockerfile = `FROM busybox:1.36
+RUN mkdir -p /www && echo studio > /www/index.html
+HEALTHCHECK --interval=5s --timeout=2s --retries=2 CMD wget -q -O /dev/null http://localhost:3000/ || exit 1
+CMD addr=$(grep -w "$HOSTNAME" /etc/hosts | head -n 1 | cut -f 1); exec httpd -f -p "${addr:-$HOSTNAME}:3000" -h /www
+`
+
+// What the bed proves is what the change touches: a long-running container whose declared check or image
+// changed, or that is new; never a step, never one left as it was.
+func TestTheBedProvesWhatTheChangeTouches(t *testing.T) {
+	root := t.TempDir()
+	git := func(args ...string) {
+		t.Helper()
+		cmd := exec.Command("git", append([]string{"-C", root, "-c", "user.email=lab@example", "-c", "user.name=lab"}, args...)...)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			t.Fatalf("git %v: %v %s", args, err, out)
+		}
+	}
+	write := func(module, body string) {
+		t.Helper()
+		if err := os.MkdirAll(filepath.Join(root, "modules", module), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(root, "modules", module, "module.json"), []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	git("init", "-q", "-b", "main")
+	write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:a","health":{"kind":"runtime"}},
+	  {"id":"seed","type":"container","image":"web@sha256:a","run-once":true,"health":{"kind":"runtime"}}]}`)
+	write("db", `{"module":"db","resources":[{"id":"server","type":"container","image":"db@sha256:a","health":{"kind":"runtime"}}]}`)
+	git("add", "-A")
+	git("commit", "-qm", "base")
+	write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:b","health":{"kind":"runtime"}},
+	  {"id":"seed","type":"container","image":"web@sha256:b","run-once":true,"health":{"kind":"runtime"}}]}`)
+	write("cache", `{"module":"cache","resources":[{"id":"server","type":"container","image":"cache@sha256:a","health":{"kind":"tcp","endpoint":"redis"}},
+	  {"id":"plain","type":"container","image":"cache@sha256:a"}]}`)
+	show := func(ref, path string) ([]byte, error) {
+		return exec.Command("git", "-C", root, "show", ref+":"+path).Output()
+	}
+	changed, err := ChangedHealth(root, "main", show)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, r := range changed {
+		got = append(got, r.Module+"."+r.ID)
+	}
+	if strings.Join(got, ",") != "cache.server,web.server" {
+		t.Fatalf("the bed would prove %v; want the new module's checked container and the one whose image moved", got)
+	}
+}
@@ -1,6 +1,7 @@
 package replays

 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"encoding/binary"
@@ -74,11 +75,21 @@ func (d *Docker) Pull(ctx context.Context, image string) error {
 	if err := d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil); err == nil {
 		return nil
 	}
-	name, tag, _ := strings.Cut(image, ":")
-	if tag == "" {
-		tag = "latest"
+	// A digest is the tag the runtime is asked for, as its own client asks: `name@sha256:…` cut at the
+	// `@`; otherwise the tag after the last `:` that is not part of a registry's port.
+	name, tag, digested := strings.Cut(image, "@")
+	if !digested {
+		name, tag = image, "latest"
+		if at := strings.LastIndex(image, ":"); at > strings.LastIndex(image, "/") {
+			name, tag = image[:at], image[at+1:]
+		}
 	}
-	return d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil)
+	err := d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil)
+	if err != nil {
+		return err
+	}
+	// The runtime answers a pull it could not make with a stream that ends in an error, not a status.
+	return d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil)
 }

 // Network makes a network of its own and answers its id.
@@ -210,3 +221,41 @@ func (d *Docker) Exec(ctx context.Context, id string, cmd ...string) (int, error
 	}
 	return -1, fmt.Errorf("%v did not end", cmd)
 }
+
+// Build builds an image from a Dockerfile alone, tagged as given, and labelled so a replay that dies is
+// cleaned up by it.
+func (d *Docker) Build(ctx context.Context, tag, dockerfile string) error {
+	var archive bytes.Buffer
+	tw := tar.NewWriter(&archive)
+	if err := tw.WriteHeader(&tar.Header{Name: "Dockerfile", Mode: 0o644, Size: int64(len(dockerfile))}); err != nil {
+		return err
+	}
+	if _, err := tw.Write([]byte(dockerfile)); err != nil {
+		return err
+	}
+	if err := tw.Close(); err != nil {
+		return err
+	}
+	labels, _ := json.Marshal(map[string]string{d.Label: "1"})
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://docker/build?t="+url.QueryEscape(tag)+
+		"&labels="+url.QueryEscape(string(labels))+"&rm=1", &archive)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-tar")
+	res, err := d.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	said, _ := io.ReadAll(res.Body)
+	if res.StatusCode >= 300 || bytes.Contains(said, []byte(`"error"`)) {
+		return fmt.Errorf("building %s: %s %s", tag, res.Status, strings.TrimSpace(string(said)))
+	}
+	return nil
+}
+
+// RemoveImage removes an image.
+func (d *Docker) RemoveImage(ctx context.Context, ref string) {
+	_ = d.call(ctx, http.MethodDelete, "/images/"+url.PathEscape(ref)+"?force=1", nil, nil)
+}