Count a platform manifest as its index's, and list the unrecorded copies of upstream images

The store's tools counted each platform of an upstream image apart from its index, so a
kept copy read as unrecorded and a let-go index's bytes were never counted as freed. The
copies no record names are now listed for the controller's mirrors verb, and a dry run
says what stays yet names a platform collection would take (hq ADR 0257, issue 321).

modules/artifact-store-tools/README.md

@@ -50,6 +50,18 @@ it lists. The store's collector removes every blob no manifest marks, so these n
 | `named-document` | a one-layer manifest the controller keeps under a tag (hq to-be 45 §9) | no |
 | `unrecorded` | no record names it | **never**, by any tool (ADR 0189 §3) |
 
+**A platform manifest takes the state of the recorded index naming it** (hq ADR 0257). An upstream
+image is an index over several platforms, and each platform manifest is a manifest of the repository in
+its own right; the controller lets an index go with the platform manifests only it names, so they are
+counted as the index's. Two indexes naming one platform give it the more keeping state. An index no
+record names leaves its platform manifests unrecorded with it.
+
+**Unrecorded copies of upstream images** are listed in `unrecorded_mirrors`: each manifest at the top of
+a repository only the build's mirror writes (`upstream/<host>/<path>`, or a module's `<module>/on-<argument>`
+from before hq ADR 0257) that no record names, with what it and its platforms mark alone. They are the
+input to the controller's `mirrors` verb, which records them as the mesh's on a person's word; the sweep
+then decides about them as about every other copy. This tool removes none of them.
+
 The records come from the controller's `artifacts` verb, asked with the references already let go of;
 when that answer is too large to carry, it is asked without them and the answer says so.
 
@@ -60,6 +72,11 @@ lets go of (ADR 0189 §2), so a real run asks its `collect` with `why` and `conf
 every kept archive first and records the run as a hand-act. Bytes come back at the store's nightly
 collection, which runs with the store held still.
 
+An index goes with the platform manifests it names that no kept index of its repository also names, as
+the controller's sweep lets them go (hq ADR 0257). The controller reads only its records; the store's
+files show what else names a platform, so a dry run lists in `platforms_named_by_what_stays` every
+manifest that stays and would lose a platform it names — to be read before a real run.
+
 ## Tests
 
 ```

modules/artifact-store-tools/cmd/artifact-store-tools/classify.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"regexp"
 	"sort"
 )
 
@@ -29,6 +30,23 @@ type Entry struct {
 	Bytes int64 `json:"bytes"`
 	// Unread says its content could not be read, so what it marks is not known beyond itself.
 	Unread string `json:"unread,omitempty"`
+	// Index is the index of the same repository whose state a platform manifest takes (novox/hq ADR
+	// 0257): the controller lets an index go with the platform manifests it names, so they are its.
+	Index string `json:"index,omitempty"`
+	// Mirror says it is an unrecorded copy at the top of a repository the mirror writes — what the
+	// controller's mirrors verb may record as the mesh's (novox/hq ADR 0257).
+	Mirror bool `json:"mirror,omitempty"`
+}
+
+// mirrorRepository is a repository only the build's mirror writes: one upstream image's repository, or a
+// module's own repository of a base from before ADR 0257. The controller's mirrors verb takes the same.
+var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)
+
+// keeping orders the states a platform manifest may take from the indexes naming it: the most keeping
+// wins, so a platform two indexes name is as kept as the more kept of them.
+var keeping = map[string]int{
+	StateKept: 0, StateNamedDocument: 1, StateHolderKept: 2, StateEligible: 3, StateHolderEligible: 4,
+	StateCollectedPresent: 5,
 }
 
 // recordIndex finds a record by repository and digest.
@@ -84,7 +102,55 @@ func Classify(v *View, recs *Records) []Entry {
 			out = append(out, e)
 		}
 	}
-	return out
+	return platformsOf(v, out)
+}
+
+// platformsOf gives each unrecorded platform manifest the state of the recorded index of its repository
+// that names it, the most keeping when several do, and marks each unrecorded copy at the top of a mirror
+// repository. An index no record names leaves its platform manifests unrecorded with it.
+func platformsOf(v *View, entries []Entry) []Entry {
+	at := map[string]int{}
+	for i, e := range entries {
+		at[Key(e.Repository, e.Digest)] = i
+	}
+	named := map[string]bool{}
+	for _, e := range entries {
+		m := v.M[Key(e.Repository, e.Digest)]
+		if m == nil {
+			continue
+		}
+		for _, c := range m.Children {
+			named[Key(e.Repository, c)] = true
+		}
+	}
+	for _, e := range entries {
+		rank, recorded := keeping[e.State]
+		if !recorded {
+			continue
+		}
+		m := v.M[Key(e.Repository, e.Digest)]
+		if m == nil {
+			continue
+		}
+		for _, c := range m.Children {
+			i, held := at[Key(e.Repository, c)]
+			if !held {
+				continue
+			}
+			child := &entries[i]
+			if current, was := keeping[child.State]; child.State != StateUnrecorded && (!was || current <= rank) {
+				continue
+			}
+			child.State, child.Why, child.Index = e.State, e.Why, e.Digest
+		}
+	}
+	for i := range entries {
+		e := &entries[i]
+		if e.State == StateUnrecorded && !named[Key(e.Repository, e.Digest)] && mirrorRepository.MatchString(e.Repository) {
+			e.Mirror = true
+		}
+	}
+	return entries
 }
 
 func stateOf(rec *Record, holder bool) string {
@@ -177,8 +243,10 @@ func Summarise(v *View, entries []Entry, all []Entry) map[string]*StateSum {
 }
 
 // LetGoKeys are the manifests the store would no longer hold once the controller lets go of these
-// references: an image's manifest, and an archive's holder.
+// references: an image's manifest and, for an index, the platform manifests it names that no kept index
-func LetGoKeys(v *View, refs []string) map[string]bool {
+// of the repository names too (novox/hq ADR 0257); and an archive's holder. Spared is the manifests a
+// kept index names, by Key.
+func LetGoKeys(v *View, refs []string, spared map[string]bool) map[string]bool {
 	out := map[string]bool{}
 	for _, ref := range refs {
 		repo, digest, kind := splitReference(ref)
@@ -190,6 +258,13 @@ func LetGoKeys(v *View, refs []string) map[string]bool {
 		case "image":
 			if r.Revisions[digest] {
 				out[Key(repo, digest)] = true
+				if m := v.M[Key(repo, digest)]; m != nil {
+					for _, c := range m.Children {
+						if r.Revisions[c] && !spared[Key(repo, c)] {
+							out[Key(repo, c)] = true
+						}
+					}
+				}
 			}
 		case "archive":
 			for d := range r.Revisions {
@@ -201,3 +276,99 @@ func LetGoKeys(v *View, refs []string) map[string]bool {
 	}
 	return out
 }
+
+// Spared is every manifest the controller's sweep leaves in place when it lets an index go: each kept
+// manifest, and every platform manifest a kept index names (novox/hq ADR 0257).
+func Spared(v *View, entries []Entry) map[string]bool {
+	out := map[string]bool{}
+	for _, e := range entries {
+		if e.State != StateKept || e.Index != "" {
+			continue
+		}
+		out[Key(e.Repository, e.Digest)] = true
+		if m := v.M[Key(e.Repository, e.Digest)]; m != nil {
+			for _, c := range m.Children {
+				out[Key(e.Repository, c)] = true
+			}
+		}
+	}
+	return out
+}
+
+// Orphaned are the manifests that stay and name a platform manifest collection would remove: what the
+// controller cannot see, because it reads only the records, said before anybody runs it for real.
+func Orphaned(v *View, gone map[string]bool) []string {
+	var out []string
+	for _, name := range v.L.RepoNames() {
+		for d := range v.L.Repos[name].Revisions {
+			if gone[Key(name, d)] {
+				continue
+			}
+			if m := v.M[Key(name, d)]; m != nil {
+				for _, c := range m.Children {
+					if gone[Key(name, c)] {
+						out = append(out, Key(name, d))
+						break
+					}
+				}
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// UnrecordedMirror is one unrecorded copy at the top of a mirror repository, with what it and its
+// platform manifests mark that nothing else does.
+type UnrecordedMirror struct {
+	Reference string `json:"reference"`
+	Bytes     int64  `json:"bytes"`
+	OnlyTheir int64  `json:"only_their_bytes"`
+}
+
+// UnrecordedMirrors are the copies the controller's mirrors verb may record, largest first.
+func UnrecordedMirrors(v *View, entries []Entry) []UnrecordedMirror {
+	marks := map[string]int{}
+	for name, r := range v.L.Repos {
+		for d := range r.Revisions {
+			for _, b := range v.Marks(name, d) {
+				marks[b]++
+			}
+		}
+	}
+	var out []UnrecordedMirror
+	for _, e := range entries {
+		if !e.Mirror {
+			continue
+		}
+		family := map[string]bool{}
+		own := map[string]int{}
+		add := func(digest string) {
+			for _, b := range v.Marks(e.Repository, digest) {
+				family[b] = true
+				own[b]++
+			}
+		}
+		add(e.Digest)
+		if m := v.M[Key(e.Repository, e.Digest)]; m != nil {
+			for _, c := range m.Children {
+				add(c)
+			}
+		}
+		only := map[string]bool{}
+		for b, n := range own {
+			if marks[b] == n {
+				only[b] = true
+			}
+		}
+		out = append(out, UnrecordedMirror{Reference: "artifact-store://" + e.Repository + "@" + e.Digest,
+			Bytes: v.Bytes(family), OnlyTheir: v.Bytes(only)})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].OnlyTheir != out[j].OnlyTheir {
+			return out[i].OnlyTheir > out[j].OnlyTheir
+		}
+		return out[i].Reference < out[j].Reference
+	})
+	return out
+}

modules/artifact-store-tools/cmd/artifact-store-tools/platforms_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+// A copy of an upstream image is an index and the platform manifests it names, each a manifest of the
+// repository in its own right (novox/hq ADR 0257). The records name the index; its platforms are its.
+
+// mirrorWorld is two copies of golang in one image's repository and one module's old copy:
+//
+//	upstream/docker.io/library/golang: i-old (eligible) names p-amd and p-arm; i-new (kept) names p-arm and p-new
+//	route-proxy/on-go_base: i-pre, no record, names p-pre; and a platform manifest no index names (p-lost)
+func mirrorWorld() *View {
+	const up = "upstream/docker.io/library/golang"
+	const pre = "route-proxy/on-go_base"
+	l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}}
+	v := &View{L: l, M: map[string]*Manifest{}, Unread: map[string]string{}}
+	platform := func(repo, name string, size int64) {
+		l.repo(repo).Revisions[d(name)] = true
+		l.Blobs[d(name)] = 1
+		l.Blobs[d(name+"-layer")] = size
+		v.M[Key(repo, d(name))] = &Manifest{Refs: []string{d(name + "-layer")}, Layers: []string{d(name + "-layer")}}
+	}
+	idx := func(repo, name string, children ...string) {
+		l.repo(repo).Revisions[d(name)] = true
+		l.Blobs[d(name)] = 1
+		var cs []string
+		for _, c := range children {
+			cs = append(cs, d(c))
+		}
+		v.M[Key(repo, d(name))] = &Manifest{Children: cs}
+	}
+	platform(up, "p-amd", 100)
+	platform(up, "p-arm", 200)
+	platform(up, "p-new", 300)
+	idx(up, "i-old", "p-amd", "p-arm")
+	idx(up, "i-new", "p-arm", "p-new")
+	platform(pre, "p-pre", 1000)
+	platform(pre, "p-lost", 50)
+	idx(pre, "i-pre", "p-pre")
+	return v
+}
+
+func mirrorRecords() *Records {
+	const up = "artifact-store://upstream/docker.io/library/golang@"
+	return &Records{References: []Record{
+		{Reference: up + d("i-old"), State: "eligible"},
+		{Reference: up + d("i-new"), State: "kept", Why: []string{"stood-on"}},
+	}}
+}
+
+func stateByName(entries []Entry) map[string]Entry {
+	out := map[string]Entry{}
+	for _, e := range entries {
+		out[e.Repository+" "+e.Digest] = e
+	}
+	return out
+}
+
+func TestAPlatformManifestTakesTheStateOfTheIndexNamingIt(t *testing.T) {
+	v := mirrorWorld()
+	recs := mirrorRecords()
+	for i := range recs.References {
+		fill(&recs.References[i])
+	}
+	got := stateByName(Classify(v, recs))
+	up := "upstream/docker.io/library/golang "
+	for name, want := range map[string]string{
+		up + d("p-amd"):                         StateEligible, // only the eligible index names it
+		up + d("p-arm"):                         StateKept,     // both do: the more keeping wins
+		up + d("p-new"):                         StateKept,
+		"route-proxy/on-go_base " + d("p-pre"):  StateUnrecorded, // its index has no record
+		"route-proxy/on-go_base " + d("p-lost"): StateUnrecorded,
+	} {
+		if got[name].State != want {
+			t.Errorf("%s reads %s, want %s", name, got[name].State, want)
+		}
+	}
+	if got[up+d("p-arm")].Index != d("i-new") {
+		t.Errorf("p-arm names its index as %q", got[up+d("p-arm")].Index)
+	}
+	// Only the top of an unrecorded copy is offered to the mirrors verb: its index, and a platform
+	// manifest no index names.
+	var mirrors []string
+	for _, e := range got {
+		if e.Mirror {
+			mirrors = append(mirrors, e.Digest)
+		}
+	}
+	if len(mirrors) != 2 || !got["route-proxy/on-go_base "+d("i-pre")].Mirror || !got["route-proxy/on-go_base "+d("p-lost")].Mirror {
+		t.Errorf("offered as unrecorded copies: %v", mirrors)
+	}
+	um := UnrecordedMirrors(v, Classify(v, recs))
+	if len(um) != 2 || um[0].Reference != "artifact-store://route-proxy/on-go_base@"+d("i-pre") || um[0].OnlyTheir != 1+1+1000 {
+		t.Errorf("unrecorded copies: %+v", um)
+	}
+}
+
+func TestLettingAnIndexGoTakesItsOwnPlatformsAndSaysWhatStaysAndNamesOne(t *testing.T) {
+	v := mirrorWorld()
+	recs := mirrorRecords()
+	for i := range recs.References {
+		fill(&recs.References[i])
+	}
+	entries := Classify(v, recs)
+	old := "artifact-store://upstream/docker.io/library/golang@" + d("i-old")
+	gone := LetGoKeys(v, []string{old}, Spared(v, entries))
+	up := "upstream/docker.io/library/golang"
+	want := map[string]bool{Key(up, d("i-old")): true, Key(up, d("p-amd")): true}
+	if !reflect.DeepEqual(gone, want) {
+		t.Fatalf("would go: %v; want the old index and the platform only it names", gone)
+	}
+	if o := Orphaned(v, gone); len(o) != 0 {
+		t.Fatalf("what stays names something that goes: %v", o)
+	}
+	// Spared nothing, the shared platform goes too, and the kept index that names it is said.
+	all := LetGoKeys(v, []string{old}, nil)
+	if o := Orphaned(v, all); len(o) != 1 || o[0] != Key(up, d("i-new")) {
+		t.Fatalf("a kept index losing a platform was not said: %v", o)
+	}
+	out := Collected(v, &CollectAnswer{DryRun: true, WouldLetGo: []string{old}, Eligible: 1}, true, entries)
+	if out["manifests_gone"] != 2 {
+		t.Fatalf("collect said %v manifests go", out["manifests_gone"])
+	}
+	if out["collector_frees_after_bytes"].(int64)-out["collector_frees_now_bytes"].(int64) != 1+1+100 {
+		t.Fatalf("collect frees %v more, want the index, its own platform and that platform's layer",
+			out["collector_frees_after_bytes"].(int64)-out["collector_frees_now_bytes"].(int64))
+	}
+}

modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go

@@ -256,7 +256,7 @@ func TestBytesFreedCountASharedBlobAsKept(t *testing.T) {
 	v := view(t, world())
 	a := &CollectAnswer{DryRun: true, Eligible: 2, WouldLetGo: []string{
 		"artifact-store://app/server@" + d("m-old"), "artifact-store://app/tools/blobs/" + d("a-old")}}
-	out := Collected(v, a, true)
+	out := Collected(v, a, true, nil)
 	now, after := out["collector_frees_now_bytes"].(int64), out["collector_frees_after_bytes"].(int64)
 	// m-old's own content, old-only and a-old and its holder go; shared and cfg stay with m-kept.
 	want := int64(7000) + 500 + 4000 + v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")]

modules/artifact-store-tools/cmd/artifact-store-tools/tools.go

@@ -55,9 +55,11 @@ func Tools(s *Store, c Controller) []stdio.Tool {
 			Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " +
 				"names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " +
 				"eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " +
-				"a named document the controller keeps under a tag, or unrecorded — no record names it. Counted and sized by " +
+				"a named document the controller keeps under a tag, or unrecorded — no record names it. A platform manifest " +
+				"takes the state of the recorded index naming it. Counted and sized by " +
 				"state and by repository; each manifest listed when one repository is asked. Then what the records keep that " +
-				"the store does not hold. Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)",
+				"the store does not hold, and the unrecorded copies of upstream images the controller's mirrors verb may " +
+				"record (novox/hq ADR 0257). Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)",
 			Input: map[string]any{"repository": repoArg},
 			Run: func(args map[string]any) (any, error) {
 				repo := text(args, "repository")
@@ -108,7 +110,13 @@ func Tools(s *Store, c Controller) []stdio.Tool {
 				if err != nil {
 					return map[string]any{"controller": answer, "bytes_not_worked_out": err.Error()}, nil
 				}
-				return Collected(v, answer, dry), nil
+				// What a kept index names stays when an index goes (novox/hq ADR 0257): read from the
+				// records, as the controller's sweep reads it.
+				recs, err := c.Artifacts("")
+				if err != nil {
+					return map[string]any{"controller": answer, "bytes_not_worked_out": "the records could not be read: " + err.Error()}, nil
+				}
+				return Collected(v, answer, dry, Classify(v, recs)), nil
 			},
 		},
 	}
@@ -244,10 +252,20 @@ func References(v *View, recs *Records, only string) (map[string]any, error) {
 	if len(missing) > 0 {
 		said = append(said, fmt.Sprintf("%d references the records keep are not in the store", len(missing)))
 	}
+	mirrors := UnrecordedMirrors(v, entries)
+	if len(mirrors) > 0 {
+		var only int64
+		for _, m := range mirrors {
+			only += m.OnlyTheir
+		}
+		said = append(said, fmt.Sprintf("%d unrecorded copies of upstream images, %s marked by them alone: the "+
+			"controller's mirrors verb may record them as the mesh's (novox/hq ADR 0257), listed in unrecorded_mirrors",
+			len(mirrors), human(only)))
+	}
 	out := map[string]any{
 		"store": recs.Store, "kept_builds": recs.KeptBuilds, "records": recs.Counts,
 		"states": whole, "repositories": repos, "missing": nonNil(missing), "said": said,
-		"removable": collectionRemovesThese,
+		"removable": collectionRemovesThese, "unrecorded_mirrors": mirrorsOrEmpty(mirrors),
 	}
 	if only != "" {
 		out["manifests"] = byRepo[only]
@@ -262,12 +280,13 @@ func References(v *View, recs *Records, only string) (map[string]any, error) {
 }
 
 // Collected answers artifact_store_collect: the controller's answer, and what the store's collector frees.
-func Collected(v *View, a *CollectAnswer, dry bool) map[string]any {
+func Collected(v *View, a *CollectAnswer, dry bool, entries []Entry) map[string]any {
 	refs := a.LetGo
 	if dry {
 		refs = a.WouldLetGo
 	}
-	gone := LetGoKeys(v, refs)
+	gone := LetGoKeys(v, refs, Spared(v, entries))
+	orphaned := Orphaned(v, gone)
 	now, nowBlobs := v.Unmarked(nil)
 	after, afterBlobs := v.Unmarked(gone)
 	said := []string{}
@@ -282,6 +301,10 @@ func Collected(v *View, a *CollectAnswer, dry bool) map[string]any {
 	said = append(said,
 		fmt.Sprintf("the nightly collector frees %s tonight as the store is now", human(now)),
 		fmt.Sprintf("and %s once those %d manifests are gone (%s more)", human(after), len(gone), human(after-now)))
+	if len(orphaned) > 0 {
+		said = append(said, fmt.Sprintf("%d manifests that stay name a platform manifest this removes, and would "+
+			"lose it: listed in platforms_named_by_what_stays — read them before a real run", len(orphaned)))
+	}
 	if dry && a.Eligible > len(a.WouldLetGo) {
 		said = append(said, fmt.Sprintf("the controller listed %d of %d eligible: the bytes cover only those listed", len(a.WouldLetGo), a.Eligible))
 	}
@@ -289,7 +312,7 @@ func Collected(v *View, a *CollectAnswer, dry bool) map[string]any {
 		"dry_run": dry, "controller": a,
 		"collector_frees_now_bytes": now, "collector_frees_now_blobs": nowBlobs,
 		"collector_frees_after_bytes": after, "collector_frees_after_blobs": afterBlobs,
-		"manifests_gone": len(gone), "said": said,
+		"manifests_gone": len(gone), "said": said, "platforms_named_by_what_stays": nonNil(orphaned),
 		"note": "a manifest let go of frees no bytes until the store's nightly collector runs with the store held still (ADR 0189 §4)",
 	}
 	if u := unreadNote(v); u != nil {
@@ -298,6 +321,13 @@ func Collected(v *View, a *CollectAnswer, dry bool) map[string]any {
 	return out
 }
 
+func mirrorsOrEmpty(m []UnrecordedMirror) []UnrecordedMirror {
+	if m == nil {
+		return []UnrecordedMirror{}
+	}
+	return m
+}
+
 func nonNil(s []string) []string {
 	if s == nil {
 		return []string{}