Count a platform manifest as its index's, and list the unrecorded copies of upstream images

The store's tools counted each platform of an upstream image apart from its index, so a kept copy read as unrecorded and a let-go index's bytes were never counted as freed. The copies no record names are now listed for the controller's mirrors verb, and a dry run says what stays yet names a platform collection would take (hq ADR 0257, issue 321).
2026-10-08 14:11:02 +02:00
parent c78b5fc9f5
commit 2dffc48c81
5 changed files with 360 additions and 11 deletions
@@ -1,6 +1,7 @@
 package main

 import (
+	"regexp"
 	"sort"
 )

@@ -29,6 +30,23 @@ type Entry struct {
 	Bytes int64 `json:"bytes"`
 	// Unread says its content could not be read, so what it marks is not known beyond itself.
 	Unread string `json:"unread,omitempty"`
+	// Index is the index of the same repository whose state a platform manifest takes (novox/hq ADR
+	// 0257): the controller lets an index go with the platform manifests it names, so they are its.
+	Index string `json:"index,omitempty"`
+	// Mirror says it is an unrecorded copy at the top of a repository the mirror writes — what the
+	// controller's mirrors verb may record as the mesh's (novox/hq ADR 0257).
+	Mirror bool `json:"mirror,omitempty"`
+}
+
+// mirrorRepository is a repository only the build's mirror writes: one upstream image's repository, or a
+// module's own repository of a base from before ADR 0257. The controller's mirrors verb takes the same.
+var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)
+
+// keeping orders the states a platform manifest may take from the indexes naming it: the most keeping
+// wins, so a platform two indexes name is as kept as the more kept of them.
+var keeping = map[string]int{
+	StateKept: 0, StateNamedDocument: 1, StateHolderKept: 2, StateEligible: 3, StateHolderEligible: 4,
+	StateCollectedPresent: 5,
 }

 // recordIndex finds a record by repository and digest.
@@ -84,7 +102,55 @@ func Classify(v *View, recs *Records) []Entry {
 			out = append(out, e)
 		}
 	}
-	return out
+	return platformsOf(v, out)
+}
+
+// platformsOf gives each unrecorded platform manifest the state of the recorded index of its repository
+// that names it, the most keeping when several do, and marks each unrecorded copy at the top of a mirror
+// repository. An index no record names leaves its platform manifests unrecorded with it.
+func platformsOf(v *View, entries []Entry) []Entry {
+	at := map[string]int{}
+	for i, e := range entries {
+		at[Key(e.Repository, e.Digest)] = i
+	}
+	named := map[string]bool{}
+	for _, e := range entries {
+		m := v.M[Key(e.Repository, e.Digest)]
+		if m == nil {
+			continue
+		}
+		for _, c := range m.Children {
+			named[Key(e.Repository, c)] = true
+		}
+	}
+	for _, e := range entries {
+		rank, recorded := keeping[e.State]
+		if !recorded {
+			continue
+		}
+		m := v.M[Key(e.Repository, e.Digest)]
+		if m == nil {
+			continue
+		}
+		for _, c := range m.Children {
+			i, held := at[Key(e.Repository, c)]
+			if !held {
+				continue
+			}
+			child := &entries[i]
+			if current, was := keeping[child.State]; child.State != StateUnrecorded && (!was || current <= rank) {
+				continue
+			}
+			child.State, child.Why, child.Index = e.State, e.Why, e.Digest
+		}
+	}
+	for i := range entries {
+		e := &entries[i]
+		if e.State == StateUnrecorded && !named[Key(e.Repository, e.Digest)] && mirrorRepository.MatchString(e.Repository) {
+			e.Mirror = true
+		}
+	}
+	return entries
 }

 func stateOf(rec *Record, holder bool) string {
@@ -177,8 +243,10 @@ func Summarise(v *View, entries []Entry, all []Entry) map[string]*StateSum {
 }

 // LetGoKeys are the manifests the store would no longer hold once the controller lets go of these
-// references: an image's manifest, and an archive's holder.
-func LetGoKeys(v *View, refs []string) map[string]bool {
+// references: an image's manifest and, for an index, the platform manifests it names that no kept index
+// of the repository names too (novox/hq ADR 0257); and an archive's holder. Spared is the manifests a
+// kept index names, by Key.
+func LetGoKeys(v *View, refs []string, spared map[string]bool) map[string]bool {
 	out := map[string]bool{}
 	for _, ref := range refs {
 		repo, digest, kind := splitReference(ref)
@@ -190,6 +258,13 @@ func LetGoKeys(v *View, refs []string) map[string]bool {
 		case "image":
 			if r.Revisions[digest] {
 				out[Key(repo, digest)] = true
+				if m := v.M[Key(repo, digest)]; m != nil {
+					for _, c := range m.Children {
+						if r.Revisions[c] && !spared[Key(repo, c)] {
+							out[Key(repo, c)] = true
+						}
+					}
+				}
 			}
 		case "archive":
 			for d := range r.Revisions {
@@ -201,3 +276,99 @@ func LetGoKeys(v *View, refs []string) map[string]bool {
 	}
 	return out
 }
+
+// Spared is every manifest the controller's sweep leaves in place when it lets an index go: each kept
+// manifest, and every platform manifest a kept index names (novox/hq ADR 0257).
+func Spared(v *View, entries []Entry) map[string]bool {
+	out := map[string]bool{}
+	for _, e := range entries {
+		if e.State != StateKept || e.Index != "" {
+			continue
+		}
+		out[Key(e.Repository, e.Digest)] = true
+		if m := v.M[Key(e.Repository, e.Digest)]; m != nil {
+			for _, c := range m.Children {
+				out[Key(e.Repository, c)] = true
+			}
+		}
+	}
+	return out
+}
+
+// Orphaned are the manifests that stay and name a platform manifest collection would remove: what the
+// controller cannot see, because it reads only the records, said before anybody runs it for real.
+func Orphaned(v *View, gone map[string]bool) []string {
+	var out []string
+	for _, name := range v.L.RepoNames() {
+		for d := range v.L.Repos[name].Revisions {
+			if gone[Key(name, d)] {
+				continue
+			}
+			if m := v.M[Key(name, d)]; m != nil {
+				for _, c := range m.Children {
+					if gone[Key(name, c)] {
+						out = append(out, Key(name, d))
+						break
+					}
+				}
+			}
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// UnrecordedMirror is one unrecorded copy at the top of a mirror repository, with what it and its
+// platform manifests mark that nothing else does.
+type UnrecordedMirror struct {
+	Reference string `json:"reference"`
+	Bytes     int64  `json:"bytes"`
+	OnlyTheir int64  `json:"only_their_bytes"`
+}
+
+// UnrecordedMirrors are the copies the controller's mirrors verb may record, largest first.
+func UnrecordedMirrors(v *View, entries []Entry) []UnrecordedMirror {
+	marks := map[string]int{}
+	for name, r := range v.L.Repos {
+		for d := range r.Revisions {
+			for _, b := range v.Marks(name, d) {
+				marks[b]++
+			}
+		}
+	}
+	var out []UnrecordedMirror
+	for _, e := range entries {
+		if !e.Mirror {
+			continue
+		}
+		family := map[string]bool{}
+		own := map[string]int{}
+		add := func(digest string) {
+			for _, b := range v.Marks(e.Repository, digest) {
+				family[b] = true
+				own[b]++
+			}
+		}
+		add(e.Digest)
+		if m := v.M[Key(e.Repository, e.Digest)]; m != nil {
+			for _, c := range m.Children {
+				add(c)
+			}
+		}
+		only := map[string]bool{}
+		for b, n := range own {
+			if marks[b] == n {
+				only[b] = true
+			}
+		}
+		out = append(out, UnrecordedMirror{Reference: "artifact-store://" + e.Repository + "@" + e.Digest,
+			Bytes: v.Bytes(family), OnlyTheir: v.Bytes(only)})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].OnlyTheir != out[j].OnlyTheir {
+			return out[i].OnlyTheir > out[j].OnlyTheir
+		}
+		return out[i].Reference < out[j].Reference
+	})
+	return out
+}