diff --git a/replays/bed.go b/replays/bed.go new file mode 100644 index 0000000..aa43213 --- /dev/null +++ b/replays/bed.go @@ -0,0 +1,410 @@ +package replays + +import ( + "bufio" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// The bed (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): **a declaration is proved before it is trusted.** +// +// Two of the nineteen image checks the catalogue's containers ship read *unhealthy* while working in the +// mesh's hands — the studio's asked an address its program does not bind, the flow editor's read settings +// from a path the module mounted elsewhere. Read without proof, they would have put back two good builds. +// So the catalogue's merge check starts every resource whose declared check or image the change touches, +// **alone**, on a throwaway runtime, and requires its check to see the program within its grace. +// +// **What it proves is the check's wiring**: that it can see the program working — a property of the image +// and the declaration, not of the mesh. A resource is started with what the module declares and nothing of +// the mesh's: its literal environment, its arguments, its files where the module writes them in full, and an +// empty directory for every other place it mounts; no secret, no binding, no provider. So: +// +// - a check naming a provision in `needs` gets no provider here, and must only reach the program — an +// http answer of any status, a connect; +// - a program that does not stay up alone — it needs what the mesh gives it — is said as not proved here, +// and judged on the first machine's gate, never passed as proved and never failed; +// - a program that stays up while its check does not see it **fails**: that is the studio's fault. +// +// A tool check and a unit's own readiness need the mesh and a machine; they are said, not proved. + +// BedResource is one resource the bed proves: its module, its id, and the resource as the manifest says it. +type BedResource struct { + Module string + ID string + Resource map[string]any + // Listens are the module's endpoints, by name: the container's own port for each. + Listens map[string]int +} + +// BedVerdict is what the bed found of one resource. +type BedVerdict struct { + Proved bool + // Failed says the program stayed up and its check did not see it: the change's fault. + Failed bool + Said string +} + +// BedLook is how often the bed looks: often, because it proves the wiring, not the timing. +var BedLook = 2 * time.Second + +// BedFloor is the least the bed waits for a check to pass, whatever the grace: a program needs a moment to +// listen even when the module declared no grace. +var BedFloor = 30 * time.Second + +// ChangedHealth is every long-running container resource of the catalogue at root that declares `health` +// and whose resource changed against base (a git ref in that checkout): its declaration, its image, or +// anything else of it. A manifest new on this branch counts whole. Read through git; an error when the +// base cannot be read. +func ChangedHealth(root, base string, show func(ref, path string) ([]byte, error)) ([]BedResource, error) { + paths, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var out []BedResource + for _, p := range paths { + raw, err := os.ReadFile(p) + if err != nil { + return nil, err + } + rel, _ := filepath.Rel(root, p) + was := map[string]string{} + if before, err := show(base, rel); err == nil { + for _, r := range resourcesOf(before) { + was[fmt.Sprint(r["id"])] = canonical(r) + } + } + module, listens, resources := manifestParts(raw) + for _, r := range resources { + if _, declared := r["health"]; !declared || fmt.Sprint(r["type"]) != "container" || !stays(r) { + continue + } + if was[fmt.Sprint(r["id"])] == canonical(r) { + continue + } + out = append(out, BedResource{Module: module, ID: fmt.Sprint(r["id"]), Resource: r, Listens: listens}) + } + } + sort.Slice(out, func(a, b int) bool { return out[a].Module+"."+out[a].ID < out[b].Module+"."+out[b].ID }) + return out, nil +} + +func stays(r map[string]any) bool { + once, _ := r["run-once"].(bool) + return !once && r["schedule"] == nil +} + +func canonical(r map[string]any) string { + raw, _ := json.Marshal(r) + return string(raw) +} + +func resourcesOf(raw []byte) []map[string]any { + _, _, rs := manifestParts(raw) + return rs +} + +func manifestParts(raw []byte) (string, map[string]int, []map[string]any) { + var m struct { + Module string `json:"module"` + Listens []struct { + Name string `json:"name"` + Port int `json:"port"` + } `json:"listens"` + Resources []map[string]any `json:"resources"` + } + _ = json.Unmarshal(raw, &m) + listens := map[string]int{} + for _, l := range m.Listens { + if l.Name != "" { + listens[l.Name] = l.Port + } + } + return m.Module, listens, m.Resources +} + +// placeholder is anything the mesh fills in on a machine: a value holding one is not the module's alone. +var placeholder = regexp.MustCompile(`\$\{[^}]*\}`) + +// Prove starts one resource alone on the runtime and looks at it with its declared check until the check +// sees it, the program stops, or its grace (at least BedFloor) runs out. Everything it made is removed. +func (d *Docker) Prove(ctx context.Context, r BedResource, files map[string]string) BedVerdict { + h, _ := r.Resource["health"].(map[string]any) + kind := fmt.Sprint(h["kind"]) + switch kind { + case "tool", "unit": + return BedVerdict{Said: fmt.Sprintf("%s.%s: a %s check needs the mesh and a machine; judged on the first "+ + "machine's gate", r.Module, r.ID, kind)} + } + image, _ := r.Resource["image"].(string) + if image == "" || placeholder.MatchString(image) { + return BedVerdict{Said: fmt.Sprintf("%s.%s: its image is built by the mesh (%v); judged on the first machine's gate", + r.Module, r.ID, r.Resource["artifact"])} + } + if err := d.Pull(ctx, image); err != nil { + return BedVerdict{Said: fmt.Sprintf("%s.%s: its image could not be fetched here: %v", r.Module, r.ID, oneLine(err.Error()))} + } + scratch, err := os.MkdirTemp("", "mesh-bed-") + if err != nil { + return BedVerdict{Said: err.Error()} + } + defer os.RemoveAll(scratch) + // Its literal environment, and every env-file the module writes in full. + var env []string + if e, ok := r.Resource["env"].(map[string]any); ok { + for k, v := range e { + if s := fmt.Sprint(v); !placeholder.MatchString(s) { + env = append(env, k+"="+s) + } + } + } + if list, ok := r.Resource["env-file"].([]any); ok { + for _, f := range list { + if content, ok := files[fmt.Sprint(f)]; ok { + env = append(env, envLines(content)...) + } + } + } + sort.Strings(env) + // Its mounts: a file the module writes in full, or an empty place of its own. + var binds []string + if list, ok := r.Resource["volumes"].([]any); ok { + for i, v := range list { + src, dst, rest := splitMount(fmt.Sprint(v)) + if dst == "" || !strings.HasPrefix(dst, "/") { + continue + } + local := filepath.Join(scratch, "m"+strconv.Itoa(i)) + if content, ok := files[src]; ok { + if err := os.WriteFile(local, []byte(content), 0o644); err != nil { + continue + } + } else if err := os.MkdirAll(local, 0o777); err != nil { + continue + } + _ = os.Chmod(local, 0o777) + binds = append(binds, local+":"+dst+rest) + } + } + var args []string + if list, ok := r.Resource["args"].([]any); ok { + for _, a := range list { + args = append(args, fmt.Sprint(a)) + } + } + network := fmt.Sprintf("mesh-bed-%d", time.Now().UnixNano()) + netID, err := d.Network(ctx, network) + if err != nil { + return BedVerdict{Said: "the bed's network could not be made: " + err.Error()} + } + defer d.RemoveNetwork(context.Background(), netID) + + grace, _ := time.ParseDuration(fmt.Sprint(h["grace"])) + if _, said := h["grace"]; !said { + grace = 60 * time.Second + } + if grace < BedFloor { + grace = BedFloor + } + config := map[string]any{"Image": image, "Env": env, "Labels": map[string]string{d.Label: "bed"}} + if len(args) > 0 { + config["Cmd"] = args + } + switch kind { + case "exec": + config["Healthcheck"] = map[string]any{"Test": []string{"CMD-SHELL", fmt.Sprint(h["command"])}, + "Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1} + case "runtime": + // The image's own command, adopted by name: an empty Test keeps it. + config["Healthcheck"] = map[string]any{"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, + "Retries": 1} + } + config["HostConfig"] = map[string]any{"Binds": binds, "NetworkMode": network} + name := fmt.Sprintf("mesh-bed-%s-%s-%d", r.Module, r.ID, time.Now().UnixNano()) + var made struct{ ID string } + if err := d.call(ctx, http.MethodPost, "/containers/create?name="+name, config, &made); err != nil { + return BedVerdict{Said: fmt.Sprintf("%s.%s could not be made here: %v", r.Module, r.ID, oneLine(err.Error()))} + } + defer d.Remove(context.Background(), made.ID) + if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil { + return BedVerdict{Said: fmt.Sprintf("%s.%s could not be started here: %v", r.Module, r.ID, oneLine(err.Error()))} + } + + port := r.Listens[fmt.Sprint(h["endpoint"])] + needs := fmt.Sprint(h["needs"]) != "" && h["needs"] != nil + deadline := time.Now().Add(grace) + last := "it was never looked at" + for { + var seen struct { + State struct { + Running bool + Health *struct { + Status string + Log []struct{ Output string } + } + } + NetworkSettings struct { + Networks map[string]struct{ IPAddress string } + } + } + if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil { + return BedVerdict{Said: err.Error()} + } + if !seen.State.Running { + return BedVerdict{Said: fmt.Sprintf("%s.%s does not stay up alone — it needs what the mesh gives it — so its "+ + "check is judged on the first machine's gate: %s", r.Module, r.ID, oneLine(lastLine(d.Logs(ctx, made.ID))))} + } + ok := false + switch kind { + case "exec", "runtime": + hs := seen.State.Health + switch { + case hs == nil: + last = "the container carries no check: its image ships none to adopt" + case hs.Status == "healthy": + ok = true + default: + last = "the runtime says " + hs.Status + if n := len(hs.Log); n > 0 { + last += ": " + oneLine(hs.Log[n-1].Output) + } + } + case "http", "tcp": + address := "" + for _, n := range seen.NetworkSettings.Networks { + address = n.IPAddress + } + ok, last = look(ctx, kind, address, port, h, needs) + default: + return BedVerdict{Said: fmt.Sprintf("%s.%s declares a %s check the bed does not know", r.Module, r.ID, kind)} + } + if ok { + return BedVerdict{Proved: true, Said: fmt.Sprintf("%s.%s: its %s check sees the program", r.Module, r.ID, kind)} + } + if time.Now().After(deadline) || ctx.Err() != nil { + return BedVerdict{Failed: true, Said: fmt.Sprintf("%s.%s stays up and its %s check does not see it within %s: %s", + r.Module, r.ID, kind, grace, last)} + } + select { + case <-ctx.Done(): + case <-time.After(BedLook): + } + } +} + +// look is one http or tcp look at the program, from outside it, as the node-engine makes it. A check that +// needs a provider only has to reach the program: any answer, a connect. +func look(ctx context.Context, kind, address string, port int, h map[string]any, needs bool) (bool, string) { + if address == "" || port == 0 { + return false, "it has no address on the bed's network yet" + } + at := net.JoinHostPort(address, strconv.Itoa(port)) + ctx, cancel := context.WithTimeout(ctx, BedLook) + defer cancel() + if kind == "tcp" { + c, err := (&net.Dialer{}).DialContext(ctx, "tcp", at) + if err != nil { + return false, oneLine(err.Error()) + } + c.Close() + return true, "" + } + scheme, _ := h["scheme"].(string) + if scheme == "" { + scheme = "http" + } + path, _ := h["path"].(string) + if path == "" { + path = "/" + } + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+at+path, nil) + res, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + Transport: insecure()}).Do(req) + if err != nil { + return false, oneLine(err.Error()) + } + res.Body.Close() + if needs { + return true, "" + } + want, _ := h["status"].(float64) + switch { + case want != 0 && res.StatusCode != int(want): + return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, int(want)) + case want == 0 && res.StatusCode >= 400: + return false, fmt.Sprintf("answered %d", res.StatusCode) + } + return true, "" +} + +func splitMount(v string) (src, dst, rest string) { + parts := strings.SplitN(v, ":", 3) + if len(parts) < 2 { + return "", "", "" + } + if len(parts) == 3 { + rest = ":" + parts[2] + } + return parts[0], parts[1], rest +} + +func envLines(content string) []string { + var out []string + s := bufio.NewScanner(strings.NewReader(content)) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") || placeholder.MatchString(line) { + continue + } + out = append(out, line) + } + return out +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} + +// FilesOf is every file a module writes in full — its content declared, nothing the mesh fills in — by the +// path it writes it at, as the module names it: what the bed gives a container that mounts or reads it. +func FilesOf(manifest []byte) map[string]string { + _, _, rs := manifestParts(manifest) + out := map[string]string{} + for _, r := range rs { + if fmt.Sprint(r["type"]) != "file" { + continue + } + path, _ := r["path"].(string) + content, ok := r["content"].(string) + if path == "" || !ok || placeholder.MatchString(content) { + continue + } + out[path] = content + } + return out +} + +// insecure is a transport that asks whether a program answers, not whom to trust. +func insecure() *http.Transport { + return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true} +} + +// oneLine is the first line of what was said, short. +func oneLine(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + if len(line) > 300 { + line = line[:300] + "…" + } + return line +} diff --git a/replays/bed_test.go b/replays/bed_test.go new file mode 100644 index 0000000..3f90df6 --- /dev/null +++ b/replays/bed_test.go @@ -0,0 +1,161 @@ +package replays + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// **The catalogue's bed** (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): every long-running resource whose +// declared `health` or image the change touches is started alone and its check must see the program within +// its grace. Run by every merge check of the catalogue on the build seat, with the change's catalogue at +// MESH_REPLAY_CATALOGUE; by hand against the catalogue beside the lab. What changed is against +// MESH_BED_BASE (origin/main by default) in that checkout. +func TestBedProvesEveryChangedHealthCheck(t *testing.T) { + root := orDefault(os.Getenv("MESH_REPLAY_CATALOGUE"), filepath.Join("..", "..", "mesh-catalog")) + if _, err := os.Stat(filepath.Join(root, "modules")); err != nil { + t.Skipf("no catalogue at %s (MESH_REPLAY_CATALOGUE names it)", root) + } + base := orDefault(os.Getenv("MESH_BED_BASE"), "origin/main") + show := func(ref, path string) ([]byte, error) { + return exec.Command("git", "-c", "safe.directory=*", "-C", root, "show", ref+":"+path).Output() + } + if _, err := exec.Command("git", "-c", "safe.directory=*", "-C", root, "rev-parse", "--verify", base).Output(); err != nil { + // What changed cannot be told, so every declared check is proved: the bed never passes what it did + // not look at. + t.Logf("%s is not in the catalogue at %s (%v): every declared check is proved", base, root, err) + } + changed, err := ChangedHealth(root, base, show) + if err != nil { + t.Fatal(err) + } + if len(changed) == 0 { + t.Logf("no declared check or image of a long-running resource changed against %s: nothing to prove", base) + return + } + docker, ok := DockerFromEnv() + if !ok { + t.Fatalf("%d changed check(s) and no container runtime to prove them on", len(changed)) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Minute) + defer cancel() + for _, r := range changed { + manifest, _ := os.ReadFile(filepath.Join(root, "modules", r.Module, "module.json")) + v := docker.Prove(ctx, r, FilesOf(manifest)) + switch { + case v.Failed: + t.Errorf("FAILS ON THE BED: %s", v.Said) + case v.Proved: + t.Logf("proved: %s", v.Said) + default: + t.Logf("NOT PROVED HERE: %s", v.Said) + } + } +} + +// **R-studio — a check that asks an address the program does not bind fails the bed, not a machine** +// (novox/hq ADR 0240 Phase D, done when). The hosted database suite's studio binds the address the runtime +// puts in HOSTNAME, so it answered only on its network address while its image's own check asked localhost: +// unhealthy for ever while working, until the module set HOSTNAME=0.0.0.0. Adopted by name without proof, it +// would have put back a good build. +// +// The replay is that image in miniature: a web server that binds $HOSTNAME's address, and an image check that +// asks localhost. Adopted as the module declared it before the fix, the bed fails it; with the fix, it proves +// it. The image is built here and removed after, with everything the bed made. +func TestBedFailsTheStudiosFalseUnhealthy(t *testing.T) { + docker, ok := DockerFromEnv() + if !ok { + t.Skip("no container runtime: the studio is a container") + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) + defer cancel() + if err := docker.Pull(ctx, "busybox:1.36"); err != nil { + t.Fatal(err) + } + tag := fmt.Sprintf("mesh-replay-studio:%d", time.Now().UnixNano()) + if err := docker.Build(ctx, tag, StudioDockerfile); err != nil { + t.Fatal(err) + } + defer docker.RemoveImage(context.Background(), tag) + was := BedFloor + BedFloor = 15 * time.Second + defer func() { BedFloor = was }() + + studio := func(env map[string]any) BedResource { + r := map[string]any{"id": "studio", "type": "container", "name": "supabase-studio", "image": tag, + "health": map[string]any{"kind": "runtime", "grace": "10s"}} + if env != nil { + r["env"] = env + } + return BedResource{Module: "supabase", ID: "studio", Resource: r, Listens: map[string]int{}} + } + before := docker.Prove(ctx, studio(nil), nil) + if !before.Failed || !strings.Contains(before.Said, "does not see it") { + t.Fatalf("the studio's false unhealthy was not failed on the bed: %+v", before) + } + t.Logf("before the fix: %s", before.Said) + after := docker.Prove(ctx, studio(map[string]any{"HOSTNAME": "0.0.0.0"}), nil) + if !after.Proved { + t.Fatalf("the studio with HOSTNAME=0.0.0.0 was not proved: %+v", after) + } + t.Logf("with the fix: %s", after.Said) +} + +// StudioDockerfile is the studio in miniature: it serves on the address HOSTNAME names, and its own check +// asks localhost, as the studio's image does. +const StudioDockerfile = `FROM busybox:1.36 +RUN mkdir -p /www && echo studio > /www/index.html +HEALTHCHECK --interval=5s --timeout=2s --retries=2 CMD wget -q -O /dev/null http://localhost:3000/ || exit 1 +CMD addr=$(grep -w "$HOSTNAME" /etc/hosts | head -n 1 | cut -f 1); exec httpd -f -p "${addr:-$HOSTNAME}:3000" -h /www +` + +// What the bed proves is what the change touches: a long-running container whose declared check or image +// changed, or that is new; never a step, never one left as it was. +func TestTheBedProvesWhatTheChangeTouches(t *testing.T) { + root := t.TempDir() + git := func(args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", root, "-c", "user.email=lab@example", "-c", "user.name=lab"}, args...)...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v %s", args, err, out) + } + } + write := func(module, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "modules", module), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "modules", module, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + git("init", "-q", "-b", "main") + write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:a","health":{"kind":"runtime"}}, + {"id":"seed","type":"container","image":"web@sha256:a","run-once":true,"health":{"kind":"runtime"}}]}`) + write("db", `{"module":"db","resources":[{"id":"server","type":"container","image":"db@sha256:a","health":{"kind":"runtime"}}]}`) + git("add", "-A") + git("commit", "-qm", "base") + write("web", `{"module":"web","resources":[{"id":"server","type":"container","image":"web@sha256:b","health":{"kind":"runtime"}}, + {"id":"seed","type":"container","image":"web@sha256:b","run-once":true,"health":{"kind":"runtime"}}]}`) + write("cache", `{"module":"cache","resources":[{"id":"server","type":"container","image":"cache@sha256:a","health":{"kind":"tcp","endpoint":"redis"}}, + {"id":"plain","type":"container","image":"cache@sha256:a"}]}`) + show := func(ref, path string) ([]byte, error) { + return exec.Command("git", "-C", root, "show", ref+":"+path).Output() + } + changed, err := ChangedHealth(root, "main", show) + if err != nil { + t.Fatal(err) + } + var got []string + for _, r := range changed { + got = append(got, r.Module+"."+r.ID) + } + if strings.Join(got, ",") != "cache.server,web.server" { + t.Fatalf("the bed would prove %v; want the new module's checked container and the one whose image moved", got) + } +} diff --git a/replays/docker.go b/replays/docker.go index beddac6..5689294 100644 --- a/replays/docker.go +++ b/replays/docker.go @@ -1,6 +1,7 @@ package replays import ( + "archive/tar" "bytes" "context" "encoding/binary" @@ -74,11 +75,21 @@ func (d *Docker) Pull(ctx context.Context, image string) error { if err := d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil); err == nil { return nil } - name, tag, _ := strings.Cut(image, ":") - if tag == "" { - tag = "latest" + // A digest is the tag the runtime is asked for, as its own client asks: `name@sha256:…` cut at the + // `@`; otherwise the tag after the last `:` that is not part of a registry's port. + name, tag, digested := strings.Cut(image, "@") + if !digested { + name, tag = image, "latest" + if at := strings.LastIndex(image, ":"); at > strings.LastIndex(image, "/") { + name, tag = image[:at], image[at+1:] + } } - return d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil) + err := d.call(ctx, http.MethodPost, "/images/create?fromImage="+url.QueryEscape(name)+"&tag="+url.QueryEscape(tag), nil, nil) + if err != nil { + return err + } + // The runtime answers a pull it could not make with a stream that ends in an error, not a status. + return d.call(ctx, http.MethodGet, "/images/"+url.PathEscape(image)+"/json", nil, nil) } // Network makes a network of its own and answers its id. @@ -210,3 +221,41 @@ func (d *Docker) Exec(ctx context.Context, id string, cmd ...string) (int, error } return -1, fmt.Errorf("%v did not end", cmd) } + +// Build builds an image from a Dockerfile alone, tagged as given, and labelled so a replay that dies is +// cleaned up by it. +func (d *Docker) Build(ctx context.Context, tag, dockerfile string) error { + var archive bytes.Buffer + tw := tar.NewWriter(&archive) + if err := tw.WriteHeader(&tar.Header{Name: "Dockerfile", Mode: 0o644, Size: int64(len(dockerfile))}); err != nil { + return err + } + if _, err := tw.Write([]byte(dockerfile)); err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + labels, _ := json.Marshal(map[string]string{d.Label: "1"}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://docker/build?t="+url.QueryEscape(tag)+ + "&labels="+url.QueryEscape(string(labels))+"&rm=1", &archive) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/x-tar") + res, err := d.http.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + said, _ := io.ReadAll(res.Body) + if res.StatusCode >= 300 || bytes.Contains(said, []byte(`"error"`)) { + return fmt.Errorf("building %s: %s %s", tag, res.Status, strings.TrimSpace(string(said))) + } + return nil +} + +// RemoveImage removes an image. +func (d *Docker) RemoveImage(ctx context.Context, ref string) { + _ = d.call(ctx, http.MethodDelete, "/images/"+url.PathEscape(ref)+"?force=1", nil, nil) +}