Record the bases a build copies, keep them by the builds that stood on them, and copy each image once

A copied base was named only in what a build stood on, and nowhere when the build failed,
so the store's sweep could never let one go (hq issue 321). One repository per upstream
image stops each module asking the public registry for the same image again, and letting
an index go now takes its own platform manifests, which otherwise kept every byte. A
person can record the copies no record names through the new mirrors verb (hq ADR 0257).

The forge test fix is the same commit as on feat/plain-notifications: main fails without it.

internal/inventory/builds.go

@@ -39,6 +39,9 @@ type Build struct {
 	// Against is every artifact this build stood on, as references rather than module names —
 	// what makes a build edge derived rather than declared (ADR 0009).
 	Against []string
+	// Mirrored is every base this build copied into the artifact store (novox/hq ADR 0257), said
+	// whether the build worked or not: the copy is the mesh's, recorded in artifact_mirrored.
+	Mirrored []string
 	// Read is every repository this build read source from besides the module's own (novox/hq
 	// 04-ISSUES/131), at the ref it read.
 	Read []ReadRepository
@@ -120,7 +123,30 @@ func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
 		 on conflict (id) do nothing`,
 		b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
 		b.Path, manifestOrNil(b.Manifest), against, read, asked, b.SourceFingerprint)
-	return err
+	if err != nil {
+		return err
+	}
+	return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
+}
+
+// RecordMirrored keeps that the mesh copied these bases into the artifact store (novox/hq ADR 0257):
+// by the build that said so, or by a person who says why. The first record of a copy stands.
+func (i *Inventory) RecordMirrored(ctx context.Context, build, why string, references []string) error {
+	var by *string
+	if build != "" {
+		by = &build
+	}
+	for _, reference := range references {
+		if reference == "" {
+			continue
+		}
+		if _, err := i.store.Pool().Exec(ctx,
+			`insert into artifact_mirrored (reference, build, why) values ($1, $2, $3)
+			 on conflict (reference) do nothing`, reference, by, why); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Builds is what has happened lately, newest first.

internal/inventory/collection.go

@@ -37,6 +37,10 @@ const (
 	// KeptUnaddressable: it names no digest, so nothing here can speak for it, and it is kept rather
 	// than guessed about.
 	KeptUnaddressable = "unaddressable"
+	// KeptStoodOn: it is a base the mesh copied in, and one of the KeptBuilds most recent successful
+	// builds of a module the mesh still holds stood on it (novox/hq ADR 0257). A copy has no builds of
+	// its own; it is kept by the builds that used it.
+	KeptStoodOn = "stood-on"
 )
 
 // The states an artifact the mesh recorded making is in (novox/hq ADR 0251).
@@ -77,7 +81,7 @@ func (i *Inventory) Artifacts(ctx context.Context) ([]ArtifactState, error) {
 	if err != nil {
 		return nil, err
 	}
-	all, err := i.everyReferenceMade(ctx)
+	all, err := i.everyReferenceRecorded(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -156,8 +160,9 @@ func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
 	// join every module ever built kept five builds' artifacts for ever — and once the store's
 	// collector runs for real, what the keep set says is what the disk holds.
 	recent, err := i.store.Pool().Query(ctx,
-		`select made from (
-		    select b.made, row_number() over (partition by b.module order by b.at desc, b.id desc) as back
+		`select made, built_against from (
+		    select b.made, b.built_against,
+		           row_number() over (partition by b.module order by b.at desc, b.id desc) as back
 		      from build b
 		      join module m on m.name = b.module
 		     where b.failed = '' and b.module is not null and b.module <> ''
@@ -166,18 +171,26 @@ func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
 		return nil, err
 	}
 	defer recent.Close()
+	var stoodOn []string
 	for recent.Next() {
-		var raw []byte
-		if err := recent.Scan(&raw); err != nil {
+		var raw, against []byte
+		if err := recent.Scan(&raw, &against); err != nil {
 			return nil, err
 		}
 		var made []Artifact
-		if err := json.Unmarshal(raw, &made); err != nil {
-			continue
+		if err := json.Unmarshal(raw, &made); err == nil {
+			for _, a := range made {
+				if reference := asRecorded(a.Reference); reference != "" {
+					because(reference, KeptByRecentBuild)
+				}
+			}
 		}
-		for _, a := range made {
-			if reference := asRecorded(a.Reference); reference != "" {
-				because(reference, KeptByRecentBuild)
+		var bases []string
+		if err := json.Unmarshal(against, &bases); err == nil {
+			for _, b := range bases {
+				if reference := asRecorded(b); reference != "" {
+					stoodOn = append(stoodOn, reference)
+				}
 			}
 		}
 	}
@@ -185,6 +198,27 @@ func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
 		return nil, err
 	}
 
+	// **A copied base is kept by the kept builds that stood on it, and by nothing else** (novox/hq
+	// ADR 0257). It has no builds of its own to count five of, and no machine is ever handed a base,
+	// so a definition is never its reason: a definition names the image upstream, by the digest every
+	// copy of it shares, and read as a reason it would keep every module's copy for as long as any
+	// module stood on that image. A copy already let go of stays let go of.
+	mirrored, err := i.mirroredSet(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if len(mirrored) > 0 {
+		collected, err := i.alreadyCollected(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, reference := range stoodOn {
+			if mirrored[reference] && !collected[reference] {
+				because(reference, KeptStoodOn)
+			}
+		}
+	}
+
 	// And anything a manifest mentions — every reference ever made, the recent ones included, so a
 	// kept artifact carries each of its reasons and the operator reads why, not only that (novox/hq
 	// ADR 0251). A manifest holds a reference composed with the store's address or kept bare, so the
@@ -195,6 +229,9 @@ func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
 			return nil, err
 		}
 		for _, reference := range all {
+			if mirrored[reference] {
+				continue
+			}
 			digest := digestIn(reference)
 			if digest == "" {
 				// Not something the store holds by digest; nothing here can speak for it, so it
@@ -237,6 +274,72 @@ func (i *Inventory) KeptArchives(ctx context.Context) ([]string, error) {
 	return out, nil
 }
 
+// KeptImages is every image the mesh keeps, in a stated order: the manifests whose platform manifests
+// a sweep spares when it lets an index of the same repository go (novox/hq ADR 0257).
+func (i *Inventory) KeptImages(ctx context.Context) ([]string, error) {
+	keep, err := i.KeepSet(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for reference := range keep {
+		if path, ours := catalogue.InArtifactStore(reference); ours && strings.Contains(path, "@sha256:") {
+			out = append(out, reference)
+		}
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// everyReferenceRecorded is every artifact the mesh recorded putting in its store: what each
+// successful build made, and every base a build — or a person — recorded copying in (novox/hq ADR
+// 0257). Oldest first, a copy placed at the moment it was recorded.
+func (i *Inventory) everyReferenceRecorded(ctx context.Context) ([]string, error) {
+	made, err := i.everyReferenceMade(ctx)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := i.store.Pool().Query(ctx, `select reference from artifact_mirrored order by at asc, reference asc`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	seen := map[string]bool{}
+	for _, r := range made {
+		seen[r] = true
+	}
+	out := made
+	for rows.Next() {
+		var reference string
+		if err := rows.Scan(&reference); err != nil {
+			return nil, err
+		}
+		if reference = asRecorded(reference); reference != "" && !seen[reference] {
+			seen[reference] = true
+			out = append(out, reference)
+		}
+	}
+	return out, rows.Err()
+}
+
+// mirroredSet is every base the records say the mesh copied in, as recorded.
+func (i *Inventory) mirroredSet(ctx context.Context) (map[string]bool, error) {
+	rows, err := i.store.Pool().Query(ctx, `select reference from artifact_mirrored`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]bool{}
+	for rows.Next() {
+		var reference string
+		if err := rows.Scan(&reference); err != nil {
+			return nil, err
+		}
+		out[asRecorded(reference)] = true
+	}
+	return out, rows.Err()
+}
+
 // everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
 func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
 	rows, err := i.store.Pool().Query(ctx,
@@ -331,3 +434,8 @@ func (i *Inventory) MarkCollected(ctx context.Context, references []string) erro
 	}
 	return nil
 }
+
+// Mirrored is every base the records say the mesh copied into its store (novox/hq ADR 0257).
+func (i *Inventory) Mirrored(ctx context.Context) (map[string]bool, error) {
+	return i.mirroredSet(ctx)
+}

internal/inventory/dependencies.go

@@ -115,6 +115,11 @@ func dependenciesOf(entries []Entry, against map[string][]string, read map[strin
 		}
 		for _, ref := range against[name] {
 			if rest, ok := strings.CutPrefix(ref, catalogue.ArtifactStoreScheme); ok {
+				// A copy of an upstream image is no module's artifact (novox/hq ADR 0257): its
+				// repository is named for the image, and its first segment names no module.
+				if strings.HasPrefix(rest, builder.MirrorPrefix) {
+					continue
+				}
 				if base, _, found := strings.Cut(rest, "/"); found {
 					add(name, base, EdgeStandsOn)
 				}

internal/inventory/dependencies_test.go

@@ -154,3 +154,20 @@ func TestNothingStandsOnTheBuildAgentOnlyIsBuiltByIt(t *testing.T) {
 		t.Errorf("everything source-built is built by the agent: %d edges", builtBy)
 	}
 }
+
+// A copy of an upstream image is no module's artifact (novox/hq ADR 0257), even when a module happens to
+// carry the name its repository starts with.
+func TestACopyOfAnUpstreamImageIsNoEdge(t *testing.T) {
+	entries := []Entry{
+		{Manifest: catalogue.Manifest{Module: "route-proxy"}},
+		{Manifest: catalogue.Manifest{Module: "upstream"}},
+	}
+	against := map[string][]string{
+		"route-proxy": {catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@sha256:a"},
+	}
+	for _, e := range dependenciesOf(entries, against, nil) {
+		if e.From == "route-proxy" && e.To == "upstream" {
+			t.Fatalf("a copy of an upstream image was read as a module's artifact: %+v", e)
+		}
+	}
+}

internal/inventory/migrations/0081-a-mirrored-base-is-recorded.sql

@@ -0,0 +1,30 @@
+-- A base a build copies into the artifact store is recorded where it is copied (novox/hq ADR 0257).
+--
+-- A build that stands on an image published elsewhere copies it into the store first (ADR 0096, 0097),
+-- and until now nothing recorded the copy as something the mesh made: the build named it only in what
+-- it stood on, and a build that failed after copying named it nowhere. So on 2026-10-08 the store held
+-- 374 manifests of copied bases, in 27 repositories, that the sweep could never let go of, because no
+-- record said the mesh had put them there (ADR 0189 §3).
+--
+-- One row per copy, by the reference as the mesh records it. `build` is the build that first said it
+-- copied it, whether that build worked or not; `why` is a person's words when the copy was recorded by
+-- hand (the `mirrors` verb), and empty otherwise. Not a foreign key to build: a build's row may be
+-- written after its copy is, and a copy recorded by hand has no build.
+create table artifact_mirrored (
+    reference text        primary key,
+    build     text,
+    at        timestamptz not null default now(),
+    why       text        not null default ''
+);
+
+-- The copies the records already name. A build that worked kept what it stood on, and every reference
+-- into a module's own `on-<argument>` repository there is a copy that build's mirror made: only the
+-- mirror ever wrote a repository so named (ADR 0097). The earliest build to name each is its maker.
+insert into artifact_mirrored (reference, build, at)
+select distinct on (stood_on) stood_on, b.id, b.at
+  from build b,
+       jsonb_array_elements_text(case when jsonb_typeof(b.built_against) = 'array'
+                                      then b.built_against else '[]'::jsonb end) as stood_on
+ where stood_on ~ '^artifact-store://[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+@sha256:[0-9a-f]{64}$'
+ order by stood_on, b.at asc, b.id asc
+on conflict (reference) do nothing;

internal/inventory/mirrored_test.go

@@ -0,0 +1,181 @@
+package inventory
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// A base a build copies into the store is recorded where it is copied, and kept while a kept build stood
+// on it (novox/hq ADR 0257).
+
+func mirrorRef(repository string, n int) string {
+	return fmt.Sprintf("%s%s@sha256:%064x", catalogue.ArtifactStoreScheme, repository, n)
+}
+
+// builtOn records a successful build of a module that made one image and stood on these bases.
+func builtOn(t *testing.T, inv *Inventory, id, module string, n int, mirrored []string, against ...string) {
+	t.Helper()
+	b := aBuild(id, module, "")
+	b.Made = []Artifact{{Name: "app", Kind: "image", Reference: ref(module, "app", n)}}
+	b.Against = against
+	b.Mirrored = mirrored
+	if err := inv.RecordBuild(context.Background(), b); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func stateOf(t *testing.T, inv *Inventory, reference string) ArtifactState {
+	t.Helper()
+	states, err := inv.Artifacts(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, s := range states {
+		if s.Reference == reference {
+			return s
+		}
+	}
+	return ArtifactState{Reference: reference, State: "unrecorded"}
+}
+
+func TestACopyAFailedBuildMadeIsRecordedAndNothingKeepsIt(t *testing.T) {
+	inv := fresh(t)
+	golang := mirrorRef("upstream/docker.io/library/golang", 1)
+	failed := aBuild("f1", "", "the recipe fetched an undeclared image")
+	failed.Mirrored = []string{golang}
+	if err := inv.RecordBuild(context.Background(), failed); err != nil {
+		t.Fatal(err)
+	}
+	if s := stateOf(t, inv, golang); s.State != ArtifactEligible {
+		t.Fatalf("a copy only a failed build made reads %+v; want eligible", s)
+	}
+}
+
+func TestACopyIsKeptByTheKeptBuildsThatStoodOnItAndByNothingElse(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	holding(t, inv, "proxy")
+	old := mirrorRef("upstream/docker.io/library/golang", 1)
+	current := mirrorRef("upstream/docker.io/library/golang", 2)
+
+	// The first build stood on the old copy; six more on the current one. The old copy falls out of
+	// the five kept builds; the current one is stood on by all five.
+	builtOn(t, inv, "b01", "proxy", 1, []string{old}, old)
+	for i := 2; i <= 7; i++ {
+		builtOn(t, inv, fmt.Sprintf("b%02d", i), "proxy", i, []string{current}, current)
+	}
+	if s := stateOf(t, inv, old); s.State != ArtifactEligible {
+		t.Fatalf("a copy no kept build stood on reads %+v", s)
+	}
+	if s := stateOf(t, inv, current); s.State != ArtifactKept || !slices.Equal(s.Why, []string{KeptStoodOn}) {
+		t.Fatalf("a copy the kept builds stood on reads %+v", s)
+	}
+
+	// A definition naming the image upstream, by the digest every copy shares, keeps no copy.
+	m := catalogue.Manifest{Module: "proxy", Version: "2",
+		Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "GO_BASE", Image: "golang@" + strings.SplitN(old, "@", 2)[1]}}}}
+	if err := inv.RegisterModule(ctx, m, Source{Repository: "https://forge.invalid/proxy.git"}); err != nil {
+		t.Fatal(err)
+	}
+	if s := stateOf(t, inv, old); s.State != ArtifactEligible {
+		t.Fatalf("a definition naming the upstream image kept the old copy: %+v", s)
+	}
+	go_, err := inv.ToCollect(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(go_, old) || slices.Contains(go_, current) {
+		t.Fatalf("offered %v to collect", go_)
+	}
+
+	// Once let go of, a copy stays let go of, even should a kept build still name it.
+	if err := inv.MarkCollected(ctx, []string{current}); err != nil {
+		t.Fatal(err)
+	}
+	if s := stateOf(t, inv, current); s.State != ArtifactCollected {
+		t.Fatalf("a collected copy reads %+v", s)
+	}
+}
+
+func TestAModulesBaseIsNotKeptAsACopy(t *testing.T) {
+	// A base that is another module's artifact is kept by that module's own builds, as before: the
+	// stood-on reason is a copy's alone.
+	inv := fresh(t)
+	holding(t, inv, "app")
+	base := ref("go-base", "build", 1)
+	b := aBuild("g1", "go-base", "")
+	b.Made = []Artifact{{Name: "build", Kind: "image", Reference: base}}
+	if err := inv.RecordBuild(context.Background(), b); err != nil {
+		t.Fatal(err)
+	}
+	builtOn(t, inv, "a1", "app", 1, nil, base)
+	if s := stateOf(t, inv, base); s.State != ArtifactEligible {
+		t.Fatalf("a forgotten module's artifact read %+v because a build stood on it", s)
+	}
+}
+
+func TestAPersonRecordsACopyAndTheFirstRecordStands(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	copyRef := mirrorRef("route-proxy/on-go_base", 3)
+	if err := inv.RecordMirrored(ctx, "", "copied before copies were recorded", []string{copyRef}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordMirrored(ctx, "b9", "", []string{copyRef}); err != nil {
+		t.Fatal(err)
+	}
+	known, err := inv.Mirrored(ctx)
+	if err != nil || !known[copyRef] {
+		t.Fatalf("recorded copies %v %v", known, err)
+	}
+	var why string
+	if err := inv.store.Pool().QueryRow(ctx, `select why from artifact_mirrored where reference = $1`, copyRef).Scan(&why); err != nil {
+		t.Fatal(err)
+	}
+	if why != "copied before copies were recorded" {
+		t.Fatalf("the first record did not stand: %q", why)
+	}
+	if s := stateOf(t, inv, copyRef); s.State != ArtifactEligible {
+		t.Fatalf("a copy a person recorded reads %+v", s)
+	}
+}
+
+// The migration records the copies the build records already name: only a module's own on-<argument>
+// repository, and only once, by the earliest build naming it.
+func TestTheCopiesTheRecordsAlreadyNameAreRecorded(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	copyRef := mirrorRef("route-proxy/on-go_base", 1)
+	notACopy := ref("go-base", "build", 2)
+	builtOn(t, inv, "b1", "route-proxy", 1, nil, copyRef, notACopy)
+	builtOn(t, inv, "b2", "route-proxy", 2, nil, copyRef)
+	// The rows a mesh had before the table: written by builds that did not say what they copied.
+	if _, err := inv.store.Pool().Exec(ctx, `truncate artifact_mirrored`); err != nil {
+		t.Fatal(err)
+	}
+	sql, err := os.ReadFile("migrations/0081-a-mirrored-base-is-recorded.sql")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, backfill, _ := strings.Cut(string(sql), "insert into artifact_mirrored")
+	if _, err := inv.store.Pool().Exec(ctx, "insert into artifact_mirrored"+backfill); err != nil {
+		t.Fatal(err)
+	}
+	var reference, build string
+	var n int
+	if err := inv.store.Pool().QueryRow(ctx, `select count(*) from artifact_mirrored`).Scan(&n); err != nil || n != 1 {
+		t.Fatalf("%d copies recorded (%v); want the one", n, err)
+	}
+	if err := inv.store.Pool().QueryRow(ctx, `select reference, build from artifact_mirrored`).Scan(&reference, &build); err != nil {
+		t.Fatal(err)
+	}
+	if reference != copyRef || build != "b1" {
+		t.Fatalf("recorded %s by %s", reference, build)
+	}
+}