Record the bases a build copies, keep them by the builds that stood on them, and copy each image once

A copied base was named only in what a build stood on, and nowhere when the build failed,
so the store's sweep could never let one go (hq issue 321). One repository per upstream
image stops each module asking the public registry for the same image again, and letting
an index go now takes its own platform manifests, which otherwise kept every byte. A
person can record the copies no record names through the new mirrors verb (hq ADR 0257).

The forge test fix is the same commit as on feat/plain-notifications: main fails without it.

internal/builder/builder.go

@@ -77,6 +77,11 @@ type Result struct {
 	// the default, and that branch is its trunk.
 	Branches []string
 
+	// Mirrored is every base this build copied into the artifact store, as pinned (novox/hq ADR 0257):
+	// said whether the build worked or not, because the copy is in the store either way, and the
+	// records are what decide whether it stays.
+	Mirrored []string
+
 	// Source is the build's source fingerprint (source.go): what it was made from — the module's tree,
 	// the contexts' trees, the bases and toolchains by digest — hashed. Empty where the source does not
 	// pin the build. Two builds with one fingerprint are one build, whatever digests they made
@@ -106,6 +111,19 @@ type GitCredential struct {
 func Build(ctx context.Context, run Runner, publish Publisher,
 	repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
 	forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
+	// **What was copied is said whether the build worked or not** (novox/hq ADR 0257): a base is in
+	// the store from the moment it is copied, and a build that failed after copying it is the
+	// one record that it is there.
+	var mirrored []string
+	result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
+		&mirrored, seats...)
+	result.Mirrored = mirrored
+	return result, err
+}
+
+func build(ctx context.Context, run Runner, publish Publisher,
+	repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
+	forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
 	// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
 	// that hand none — tests of everything but contexts — read as they did.
 	var seatBases map[string]string
@@ -222,10 +240,22 @@ func Build(ctx context.Context, run Runner, publish Publisher,
 		// An image published elsewhere that the build stands on is copied into the mesh's own
 		// registry first, like an upstream artifact (ADR 0096), and the recipe is handed the copy.
 		// Genesis has nowhere to copy to and pulls it into this machine's store instead.
-		mirror := func(ctx context.Context, from, repository string) (string, error) {
+		mirror := func(ctx context.Context, from, repository, former string) (string, error) {
+			if m, can := publish.(BaseMirrorer); can {
+				say("bases", "copying %s into the mesh's registry as %s", from, repository)
+				reference, err := m.MirrorBase(ctx, from, repository, former)
+				if err == nil {
+					*mirrored = append(*mirrored, reference)
+				}
+				return reference, err
+			}
 			if m, can := publish.(Mirrorer); can {
-				say("bases", "copying %s into the mesh's registry", from)
-				return m.MirrorImage(ctx, from, repository)
+				say("bases", "copying %s into the mesh's registry as %s", from, repository)
+				reference, err := m.MirrorImage(ctx, from, repository)
+				if err == nil {
+					*mirrored = append(*mirrored, reference)
+				}
+				return reference, err
 			}
 			if _, err := run(ctx, tree, "docker", "pull", from); err != nil {
 				return "", fmt.Errorf("cannot fetch %s: %w", from, err)
@@ -901,7 +931,7 @@ var _ io.Writer = (*stringWriter)(nil)
 // The order is fixed so two builds of one commit invoke the same command. Returned alongside the
 // arguments is every reference they resolved to, which is what the build stood on.
 func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[string]string,
-	mirror func(ctx context.Context, from, repository string) (string, error)) ([]string, []string, error) {
+	mirror func(ctx context.Context, from, repository, former string) (string, error)) ([]string, []string, error) {
 	if manifest.Build == nil || len(manifest.Build.On) == 0 {
 		return nil, nil, nil
 	}
@@ -924,7 +954,14 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
 					"%s stands on the image %q, which is not pinned by digest. A tag is what "+
 						"somebody else can move; name it as <image>@sha256:…", manifest.Module, base.Image)
 			}
-			reference, err := mirror(ctx, base.Image, manifest.Module+"/on-"+strings.ToLower(base.Arg))
+			// **One copy per upstream image, whichever modules stand on it** (novox/hq ADR 0257). Its
+			// repository is named for the image, not the module; the module's own repository of
+			// before is offered as a source, so the move to one copy asks upstream for nothing.
+			repository, err := MirrorRepository(base.Image)
+			if err != nil {
+				return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
+			}
+			reference, err := mirror(ctx, base.Image, repository, FormerMirrorRepository(manifest.Module, base.Arg))
 			if err != nil {
 				return nil, nil, fmt.Errorf("%s stands on %s: %w", manifest.Module, base.Image, err)
 			}

internal/builder/mirror.go

@@ -27,6 +27,40 @@ type Mirrorer interface {
 	MirrorImage(ctx context.Context, from, repository string) (string, error)
 }
 
+// BaseMirrorer copies a base a build stands on into the one repository the mesh keeps for that
+// upstream image, taking what an earlier copy under `former` already holds rather than asking
+// upstream for it again (novox/hq ADR 0257).
+type BaseMirrorer interface {
+	MirrorBase(ctx context.Context, from, repository, former string) (string, error)
+}
+
+// MirrorPrefix is the namespace of the repositories a base is mirrored into: `upstream/<host>/<path>`,
+// one repository per upstream image, whichever modules stand on it (novox/hq ADR 0257).
+const MirrorPrefix = "upstream/"
+
+// MirrorRepository is the repository the mesh keeps an upstream image's copy in: the image's own
+// host and path under MirrorPrefix, so two modules standing on one image stand on one copy, and two
+// images that share a path on different hosts are never confused. Lower case and without a port's
+// colon, because a repository name allows neither.
+func MirrorRepository(from string) (string, error) {
+	where, err := parseReference(from)
+	if err != nil {
+		return "", err
+	}
+	host := strings.TrimPrefix(strings.TrimPrefix(where.base, "https://"), "http://")
+	if host == "registry-1.docker.io" {
+		host = "docker.io"
+	}
+	host = strings.ReplaceAll(host, ":", "-")
+	return strings.ToLower(MirrorPrefix + host + "/" + where.repository), nil
+}
+
+// FormerMirrorRepository is where a module's base was copied before ADR 0257: under the module's own
+// repository, one copy per module (ADR 0097). Read only, as a source of what is already held.
+func FormerMirrorRepository(module, arg string) string {
+	return module + "/on-" + strings.ToLower(arg)
+}
+
 const (
 	mediaIndexOCI       = "application/vnd.oci.image.index.v1+json"
 	mediaIndexDocker    = "application/vnd.docker.distribution.manifest.list.v2+json"
@@ -86,6 +120,9 @@ func parseReference(ref string) (upstream, error) {
 type source struct {
 	client *http.Client
 	token  string
+	// mountFrom is a repository of the mesh's own registry the source is, so a blob is mounted from
+	// it rather than read and written again.
+	mountFrom string
 }
 
 // get fetches a registry URL, answering a bearer challenge once with an anonymous token — which is
@@ -176,6 +213,15 @@ type descriptor struct {
 // under `repository`, and returns the reference the mesh will pin: this registry, the repository,
 // and the digest of the document that was put last, which is the index where there is one.
 func (r Registry) MirrorImage(ctx context.Context, from, repository string) (string, error) {
+	return r.MirrorBase(ctx, from, repository, "")
+}
+
+// MirrorBase is MirrorImage, and where `former` — a repository of this registry — already holds the
+// image by its digest, the copy is made from there: manifests read from this registry, blobs mounted
+// across rather than moved (novox/hq ADR 0257). Upstream is asked only for what no repository here
+// holds, which is what kept the public hub's anonymous pull limit out of reach when every module's
+// base moved into one shared repository.
+func (r Registry) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
 	where, err := parseReference(from)
 	if err != nil {
 		return "", err
@@ -195,6 +241,17 @@ func (r Registry) MirrorImage(ctx context.Context, from, repository string) (str
 		}
 	}
 	src := &source{client: r.client()}
+	if former != "" && former != repository && strings.HasPrefix(where.reference, "sha256:") {
+		held, err := r.has(ctx, "http://"+r.Address+"/v2/"+former+"/manifests/"+where.reference, manifestAccept)
+		if err != nil {
+			return "", fmt.Errorf("asking %s whether %s holds %s: %w", r.Address, former, from, err)
+		}
+		if held {
+			// The same bytes, already here: copied from this registry, each blob mounted.
+			where = upstream{base: "http://" + r.Address, repository: former, reference: where.reference}
+			src.mountFrom = former
+		}
+	}
 	digest, err := r.copyManifest(ctx, src, where, where.reference, repository)
 	if err != nil {
 		return "", fmt.Errorf("copying %s into %s/%s: %w", from, r.Address, repository, err)
@@ -286,16 +343,14 @@ func (r Registry) copyBlob(ctx context.Context, src *source, where upstream, dig
 	} else if there {
 		return nil
 	}
-	response, err := src.get(ctx, where.base+"/v2/"+where.repository+"/blobs/"+digest, "")
-	if err != nil {
-		return err
+	// **Mounted where this registry already holds it** (novox/hq ADR 0257): a blob is stored once
+	// whichever repositories link it, so a mount moves no bytes. A registry that cannot mount answers
+	// with an ordinary upload's location, and the blob is moved as before.
+	uploads := base + "/blobs/uploads/"
+	if src.mountFrom != "" {
+		uploads += "?mount=" + digest + "&from=" + src.mountFrom
 	}
-	defer response.Body.Close()
-	if response.StatusCode != http.StatusOK {
-		return fmt.Errorf("%s/%s: blob %s: %s", where.base, where.repository, digest, response.Status)
-	}
-
-	start, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/blobs/uploads/", nil)
+	start, err := http.NewRequestWithContext(ctx, http.MethodPost, uploads, nil)
 	if err != nil {
 		return err
 	}
@@ -304,9 +359,21 @@ func (r Registry) copyBlob(ctx context.Context, src *source, where upstream, dig
 		return fmt.Errorf("cannot start an upload to %s: %w", base, err)
 	}
 	begun.Body.Close()
+	if begun.StatusCode == http.StatusCreated && src.mountFrom != "" {
+		return nil
+	}
 	if begun.StatusCode != http.StatusAccepted {
 		return fmt.Errorf("%s answered %s when asked where to put a blob", base, begun.Status)
 	}
+
+	response, err := src.get(ctx, where.base+"/v2/"+where.repository+"/blobs/"+digest, "")
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s/%s: blob %s: %s", where.base, where.repository, digest, response.Status)
+	}
 	location := begun.Header.Get("Location")
 	if location == "" {
 		return fmt.Errorf("%s accepted an upload and said nowhere to put it", base)

internal/builder/mirror_shared_test.go

@@ -0,0 +1,245 @@
+package builder
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// One copy per upstream image, whichever modules stand on it (novox/hq ADR 0257).
+
+func TestAnUpstreamImageHasOneRepositoryNamedForIt(t *testing.T) {
+	for from, want := range map[string]string{
+		"golang@sha256:abc":                       "upstream/docker.io/library/golang",
+		"n8nio/n8n@sha256:abc":                    "upstream/docker.io/n8nio/n8n",
+		"quay.io/minio/mc@sha256:abc":             "upstream/quay.io/minio/mc",
+		"ghcr.io/Mailu/Admin@sha256:abc":          "upstream/ghcr.io/mailu/admin",
+		"localhost:5000/x/y@sha256:abc":           "upstream/localhost-5000/x/y",
+		"docker.io/library/alpine:3.20@sha256:ab": "upstream/docker.io/library/alpine",
+	} {
+		got, err := MirrorRepository(from)
+		if err != nil {
+			t.Fatalf("%s: %v", from, err)
+		}
+		if got != want {
+			t.Errorf("%s: got %q want %q", from, got, want)
+		}
+	}
+	if got := FormerMirrorRepository("route-proxy", "GO_BASE"); got != "route-proxy/on-go_base" {
+		t.Fatalf("the former repository is %q", got)
+	}
+}
+
+// aRegistryOfRepositories is a registry the way the real one is: blobs stored once, and each
+// repository linking the blobs and manifests it holds. It mounts a blob across repositories.
+type aRegistryOfRepositories struct {
+	mu        sync.Mutex
+	blobs     map[string][]byte          // digest → bytes, stored once
+	links     map[string]map[string]bool // repository → blob digests it links
+	manifests map[string][]byte          // repository@digest → document
+	uploads   int
+	mounts    int
+	gets      int
+}
+
+func newRegistryOfRepositories() *aRegistryOfRepositories {
+	return &aRegistryOfRepositories{blobs: map[string][]byte{}, links: map[string]map[string]bool{},
+		manifests: map[string][]byte{}}
+}
+
+func (m *aRegistryOfRepositories) link(repository, digest string) {
+	if m.links[repository] == nil {
+		m.links[repository] = map[string]bool{}
+	}
+	m.links[repository][digest] = true
+}
+
+// split reads /v2/<repository>/<kind>/<rest> with a repository of any depth.
+func splitPath(path string) (repository, kind, rest string) {
+	path = strings.TrimPrefix(path, "/v2/")
+	for _, k := range []string{"/manifests/", "/blobs/uploads/", "/blobs/"} {
+		if i := strings.Index(path, k); i >= 0 {
+			return path[:i], strings.Trim(k, "/"), path[i+len(k):]
+		}
+	}
+	return "", "", ""
+}
+
+func (m *aRegistryOfRepositories) handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		m.mu.Lock()
+		defer m.mu.Unlock()
+		repository, kind, rest := splitPath(r.URL.Path)
+		switch {
+		case kind == "manifests" && (r.Method == http.MethodHead || r.Method == http.MethodGet):
+			body, ok := m.manifests[repository+"@"+rest]
+			if !ok {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			if r.Method == http.MethodGet {
+				m.gets++
+				w.Header().Set("Content-Type", mediaTypeOf(body))
+				_, _ = w.Write(body)
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		case kind == "manifests" && r.Method == http.MethodPut:
+			body, _ := readAll(r)
+			m.manifests[repository+"@"+digestOf(body)] = body
+			w.WriteHeader(http.StatusCreated)
+		case kind == "blobs" && (r.Method == http.MethodHead || r.Method == http.MethodGet):
+			if !m.links[repository][rest] {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			if r.Method == http.MethodGet {
+				m.gets++
+				_, _ = w.Write(m.blobs[rest])
+				return
+			}
+			w.WriteHeader(http.StatusOK)
+		case kind == "blobs/uploads" && r.Method == http.MethodPost:
+			if digest, from := r.URL.Query().Get("mount"), r.URL.Query().Get("from"); digest != "" && m.links[from][digest] {
+				m.link(repository, digest)
+				m.mounts++
+				w.WriteHeader(http.StatusCreated)
+				return
+			}
+			w.Header().Set("Location", "/v2/"+repository+"/blobs/uploads/one")
+			w.WriteHeader(http.StatusAccepted)
+		case kind == "blobs/uploads" && r.Method == http.MethodPut:
+			body, _ := readAll(r)
+			digest := r.URL.Query().Get("digest")
+			if digestOf(body) != digest {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			m.blobs[digest] = body
+			m.link(repository, digest)
+			m.uploads++
+			w.WriteHeader(http.StatusCreated)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	})
+}
+
+func mediaTypeOf(body []byte) string {
+	switch {
+	case strings.Contains(string(body), mediaIndexOCI):
+		return mediaIndexOCI
+	default:
+		return mediaManifestOCI
+	}
+}
+
+// A base a module's own repository already holds is copied into the image's repository from there:
+// every blob mounted, nothing asked of upstream — which may be gone, or rationing anonymous pulls.
+func TestABaseHeldUnderTheModulesFormerRepositoryIsMountedNotFetchedAgain(t *testing.T) {
+	src, indexDigest, srcBlobs := anUpstreamRegistry(t)
+	dst := newRegistryOfRepositories()
+	dstServer := httptest.NewServer(dst.handler())
+	defer dstServer.Close()
+	address := strings.TrimPrefix(dstServer.URL, "http://")
+	r := Registry{Address: address, HTTP: src.Client()}
+	host := strings.TrimPrefix(src.URL, "http://")
+
+	// Before: copied the old way, under the module's repository.
+	if _, err := r.MirrorImage(context.Background(), host+"/library/thing:latest", "hello-web/on-thing_base"); err != nil {
+		t.Fatal(err)
+	}
+	uploaded := dst.uploads
+	if uploaded != len(srcBlobs) {
+		t.Fatalf("the first copy uploaded %d of %d blobs", uploaded, len(srcBlobs))
+	}
+	src.Close()
+
+	from := host + "/library/thing@" + indexDigest
+	repository, err := MirrorRepository(from)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reference, err := r.MirrorBase(context.Background(), from, repository, "hello-web/on-thing_base")
+	if err != nil {
+		t.Fatalf("a base the registry holds was asked of an upstream that is gone: %v", err)
+	}
+	if reference != address+"/"+repository+"@"+indexDigest {
+		t.Fatalf("pinned as %q", reference)
+	}
+	if dst.uploads != uploaded {
+		t.Fatalf("the move to one repository uploaded %d blobs again", dst.uploads-uploaded)
+	}
+	if dst.mounts != len(srcBlobs) {
+		t.Fatalf("%d of %d blobs mounted", dst.mounts, len(srcBlobs))
+	}
+	// The index and both platform manifests are in the image's repository, and every blob is linked.
+	for key := range dst.manifests {
+		if strings.HasPrefix(key, "hello-web/") {
+			continue
+		}
+		if !strings.HasPrefix(key, repository+"@") {
+			t.Fatalf("a manifest went to %s", key)
+		}
+	}
+	if n := countPrefix(dst.manifests, repository+"@"); n != 3 {
+		t.Fatalf("%d manifests in %s, want the index and two platforms", n, repository)
+	}
+	for digest := range srcBlobs {
+		if !dst.links[repository][digest] {
+			t.Fatalf("blob %s is not linked into %s", digest, repository)
+		}
+	}
+
+	// A second module standing on the same image: already held, nothing copied, the same reference.
+	again, err := r.MirrorBase(context.Background(), from, repository, "other-module/on-thing_base")
+	if err != nil || again != reference {
+		t.Fatalf("a second module's copy: %q %v", again, err)
+	}
+}
+
+func countPrefix(m map[string][]byte, prefix string) int {
+	n := 0
+	for k := range m {
+		if strings.HasPrefix(k, prefix) {
+			n++
+		}
+	}
+	return n
+}
+
+// What a build copied is said whether it worked or not: the copy is in the store either way, and a
+// build that failed after copying is the one record that it is there.
+func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) {
+	// A recipe that reaches for an image nobody declared is refused — after the base was copied.
+	r, workspace := aRepository(t, anImage, map[string]string{
+		"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"})
+	r.contents["modules/bus/module.json"] = anImage
+	r.tree = "aaaa"
+	m := &mirroring{recorded: r, at: "registry-a:5000"}
+	got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
+		nil, Npmrc{}, GitCredential{}, nil)
+	if err == nil {
+		t.Fatal("a recipe fetching an undeclared image was built")
+	}
+	want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64)
+	if len(got.Mirrored) != 1 || got.Mirrored[0] != want {
+		t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want)
+	}
+
+	// And a build that works says it too.
+	r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
+	r2.contents["modules/bus/module.json"] = anImage
+	r2.tree = "aaaa"
+	ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"},
+		"https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want {
+		t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want)
+	}
+}

internal/builder/standing_on_test.go

@@ -72,7 +72,7 @@ func TestAnIncompleteBaseIsRefused(t *testing.T) {
 }
 
 // noMirror is a mirror for tests whose bases are all the mesh's own.
-func noMirror(context.Context, string, string) (string, error) {
+func noMirror(context.Context, string, string, string) (string, error) {
 	return "", fmt.Errorf("nothing to copy in this test")
 }
 
@@ -86,17 +86,20 @@ func TestADeclaredVendorImageIsCopiedInAndHandedToTheRecipe(t *testing.T) {
 		},
 	}
 	var asked []string
-	args, _, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository string) (string, error) {
-		asked = append(asked, from+" -> "+repository)
+	args, _, err := standingOn(context.Background(), manifest, nil, func(_ context.Context, from, repository, former string) (string, error) {
+		asked = append(asked, from+" -> "+repository+" (from "+former+")")
 		return "127.0.0.1:5000/" + repository + "@sha256:" + strings.Repeat("d", 64), nil
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(asked) != 1 || asked[0] != "quay.io/minio/mc@sha256:"+strings.Repeat("c", 64)+" -> minio/on-mc_base" {
-		t.Fatalf("the image was not copied under the module's repository: %v", asked)
+	// One repository per upstream image, named for the image (novox/hq ADR 0257); the module's
+	// repository of before offered as where it may already be held.
+	if len(asked) != 1 || asked[0] != "quay.io/minio/mc@sha256:"+strings.Repeat("c", 64)+
+		" -> upstream/quay.io/minio/mc (from minio/on-mc_base)" {
+		t.Fatalf("the image was not copied into the image's own repository: %v", asked)
 	}
-	if strings.Join(args, " ") != "--build-arg MC_BASE=127.0.0.1:5000/minio/on-mc_base@sha256:"+strings.Repeat("d", 64) {
+	if strings.Join(args, " ") != "--build-arg MC_BASE=127.0.0.1:5000/upstream/quay.io/minio/mc@sha256:"+strings.Repeat("d", 64) {
 		t.Fatalf("the recipe was not handed the copy: %v", args)
 	}
 	// Unpinned, it is refused: a tag is what somebody else can move.