diff --git a/modules/artifact-store-tools/README.md b/modules/artifact-store-tools/README.md index 37ae51ec..c01b91b9 100644 --- a/modules/artifact-store-tools/README.md +++ b/modules/artifact-store-tools/README.md @@ -50,6 +50,18 @@ it lists. The store's collector removes every blob no manifest marks, so these n | `named-document` | a one-layer manifest the controller keeps under a tag (hq to-be 45 §9) | no | | `unrecorded` | no record names it | **never**, by any tool (ADR 0189 §3) | +**A platform manifest takes the state of the recorded index naming it** (hq ADR 0257). An upstream +image is an index over several platforms, and each platform manifest is a manifest of the repository in +its own right; the controller lets an index go with the platform manifests only it names, so they are +counted as the index's. Two indexes naming one platform give it the more keeping state. An index no +record names leaves its platform manifests unrecorded with it. + +**Unrecorded copies of upstream images** are listed in `unrecorded_mirrors`: each manifest at the top of +a repository only the build's mirror writes (`upstream//`, or a module's `/on-` +from before hq ADR 0257) that no record names, with what it and its platforms mark alone. They are the +input to the controller's `mirrors` verb, which records them as the mesh's on a person's word; the sweep +then decides about them as about every other copy. This tool removes none of them. + The records come from the controller's `artifacts` verb, asked with the references already let go of; when that answer is too large to carry, it is asked without them and the answer says so. @@ -60,6 +72,11 @@ lets go of (ADR 0189 §2), so a real run asks its `collect` with `why` and `conf every kept archive first and records the run as a hand-act. Bytes come back at the store's nightly collection, which runs with the store held still. +An index goes with the platform manifests it names that no kept index of its repository also names, as +the controller's sweep lets them go (hq ADR 0257). The controller reads only its records; the store's +files show what else names a platform, so a dry run lists in `platforms_named_by_what_stays` every +manifest that stays and would lose a platform it names — to be read before a real run. + ## Tests ``` diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/classify.go b/modules/artifact-store-tools/cmd/artifact-store-tools/classify.go index a01f3f79..49fd42ce 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/classify.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/classify.go @@ -1,6 +1,7 @@ package main import ( + "regexp" "sort" ) @@ -29,6 +30,23 @@ type Entry struct { Bytes int64 `json:"bytes"` // Unread says its content could not be read, so what it marks is not known beyond itself. Unread string `json:"unread,omitempty"` + // Index is the index of the same repository whose state a platform manifest takes (novox/hq ADR + // 0257): the controller lets an index go with the platform manifests it names, so they are its. + Index string `json:"index,omitempty"` + // Mirror says it is an unrecorded copy at the top of a repository the mirror writes — what the + // controller's mirrors verb may record as the mesh's (novox/hq ADR 0257). + Mirror bool `json:"mirror,omitempty"` +} + +// mirrorRepository is a repository only the build's mirror writes: one upstream image's repository, or a +// module's own repository of a base from before ADR 0257. The controller's mirrors verb takes the same. +var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`) + +// keeping orders the states a platform manifest may take from the indexes naming it: the most keeping +// wins, so a platform two indexes name is as kept as the more kept of them. +var keeping = map[string]int{ + StateKept: 0, StateNamedDocument: 1, StateHolderKept: 2, StateEligible: 3, StateHolderEligible: 4, + StateCollectedPresent: 5, } // recordIndex finds a record by repository and digest. @@ -84,7 +102,55 @@ func Classify(v *View, recs *Records) []Entry { out = append(out, e) } } - return out + return platformsOf(v, out) +} + +// platformsOf gives each unrecorded platform manifest the state of the recorded index of its repository +// that names it, the most keeping when several do, and marks each unrecorded copy at the top of a mirror +// repository. An index no record names leaves its platform manifests unrecorded with it. +func platformsOf(v *View, entries []Entry) []Entry { + at := map[string]int{} + for i, e := range entries { + at[Key(e.Repository, e.Digest)] = i + } + named := map[string]bool{} + for _, e := range entries { + m := v.M[Key(e.Repository, e.Digest)] + if m == nil { + continue + } + for _, c := range m.Children { + named[Key(e.Repository, c)] = true + } + } + for _, e := range entries { + rank, recorded := keeping[e.State] + if !recorded { + continue + } + m := v.M[Key(e.Repository, e.Digest)] + if m == nil { + continue + } + for _, c := range m.Children { + i, held := at[Key(e.Repository, c)] + if !held { + continue + } + child := &entries[i] + if current, was := keeping[child.State]; child.State != StateUnrecorded && (!was || current <= rank) { + continue + } + child.State, child.Why, child.Index = e.State, e.Why, e.Digest + } + } + for i := range entries { + e := &entries[i] + if e.State == StateUnrecorded && !named[Key(e.Repository, e.Digest)] && mirrorRepository.MatchString(e.Repository) { + e.Mirror = true + } + } + return entries } func stateOf(rec *Record, holder bool) string { @@ -177,8 +243,10 @@ func Summarise(v *View, entries []Entry, all []Entry) map[string]*StateSum { } // LetGoKeys are the manifests the store would no longer hold once the controller lets go of these -// references: an image's manifest, and an archive's holder. -func LetGoKeys(v *View, refs []string) map[string]bool { +// references: an image's manifest and, for an index, the platform manifests it names that no kept index +// of the repository names too (novox/hq ADR 0257); and an archive's holder. Spared is the manifests a +// kept index names, by Key. +func LetGoKeys(v *View, refs []string, spared map[string]bool) map[string]bool { out := map[string]bool{} for _, ref := range refs { repo, digest, kind := splitReference(ref) @@ -190,6 +258,13 @@ func LetGoKeys(v *View, refs []string) map[string]bool { case "image": if r.Revisions[digest] { out[Key(repo, digest)] = true + if m := v.M[Key(repo, digest)]; m != nil { + for _, c := range m.Children { + if r.Revisions[c] && !spared[Key(repo, c)] { + out[Key(repo, c)] = true + } + } + } } case "archive": for d := range r.Revisions { @@ -201,3 +276,99 @@ func LetGoKeys(v *View, refs []string) map[string]bool { } return out } + +// Spared is every manifest the controller's sweep leaves in place when it lets an index go: each kept +// manifest, and every platform manifest a kept index names (novox/hq ADR 0257). +func Spared(v *View, entries []Entry) map[string]bool { + out := map[string]bool{} + for _, e := range entries { + if e.State != StateKept || e.Index != "" { + continue + } + out[Key(e.Repository, e.Digest)] = true + if m := v.M[Key(e.Repository, e.Digest)]; m != nil { + for _, c := range m.Children { + out[Key(e.Repository, c)] = true + } + } + } + return out +} + +// Orphaned are the manifests that stay and name a platform manifest collection would remove: what the +// controller cannot see, because it reads only the records, said before anybody runs it for real. +func Orphaned(v *View, gone map[string]bool) []string { + var out []string + for _, name := range v.L.RepoNames() { + for d := range v.L.Repos[name].Revisions { + if gone[Key(name, d)] { + continue + } + if m := v.M[Key(name, d)]; m != nil { + for _, c := range m.Children { + if gone[Key(name, c)] { + out = append(out, Key(name, d)) + break + } + } + } + } + } + sort.Strings(out) + return out +} + +// UnrecordedMirror is one unrecorded copy at the top of a mirror repository, with what it and its +// platform manifests mark that nothing else does. +type UnrecordedMirror struct { + Reference string `json:"reference"` + Bytes int64 `json:"bytes"` + OnlyTheir int64 `json:"only_their_bytes"` +} + +// UnrecordedMirrors are the copies the controller's mirrors verb may record, largest first. +func UnrecordedMirrors(v *View, entries []Entry) []UnrecordedMirror { + marks := map[string]int{} + for name, r := range v.L.Repos { + for d := range r.Revisions { + for _, b := range v.Marks(name, d) { + marks[b]++ + } + } + } + var out []UnrecordedMirror + for _, e := range entries { + if !e.Mirror { + continue + } + family := map[string]bool{} + own := map[string]int{} + add := func(digest string) { + for _, b := range v.Marks(e.Repository, digest) { + family[b] = true + own[b]++ + } + } + add(e.Digest) + if m := v.M[Key(e.Repository, e.Digest)]; m != nil { + for _, c := range m.Children { + add(c) + } + } + only := map[string]bool{} + for b, n := range own { + if marks[b] == n { + only[b] = true + } + } + out = append(out, UnrecordedMirror{Reference: "artifact-store://" + e.Repository + "@" + e.Digest, + Bytes: v.Bytes(family), OnlyTheir: v.Bytes(only)}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].OnlyTheir != out[j].OnlyTheir { + return out[i].OnlyTheir > out[j].OnlyTheir + } + return out[i].Reference < out[j].Reference + }) + return out +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/platforms_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/platforms_test.go new file mode 100644 index 00000000..90846070 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/platforms_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "reflect" + "testing" +) + +// A copy of an upstream image is an index and the platform manifests it names, each a manifest of the +// repository in its own right (novox/hq ADR 0257). The records name the index; its platforms are its. + +// mirrorWorld is two copies of golang in one image's repository and one module's old copy: +// +// upstream/docker.io/library/golang: i-old (eligible) names p-amd and p-arm; i-new (kept) names p-arm and p-new +// route-proxy/on-go_base: i-pre, no record, names p-pre; and a platform manifest no index names (p-lost) +func mirrorWorld() *View { + const up = "upstream/docker.io/library/golang" + const pre = "route-proxy/on-go_base" + l := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}} + v := &View{L: l, M: map[string]*Manifest{}, Unread: map[string]string{}} + platform := func(repo, name string, size int64) { + l.repo(repo).Revisions[d(name)] = true + l.Blobs[d(name)] = 1 + l.Blobs[d(name+"-layer")] = size + v.M[Key(repo, d(name))] = &Manifest{Refs: []string{d(name + "-layer")}, Layers: []string{d(name + "-layer")}} + } + idx := func(repo, name string, children ...string) { + l.repo(repo).Revisions[d(name)] = true + l.Blobs[d(name)] = 1 + var cs []string + for _, c := range children { + cs = append(cs, d(c)) + } + v.M[Key(repo, d(name))] = &Manifest{Children: cs} + } + platform(up, "p-amd", 100) + platform(up, "p-arm", 200) + platform(up, "p-new", 300) + idx(up, "i-old", "p-amd", "p-arm") + idx(up, "i-new", "p-arm", "p-new") + platform(pre, "p-pre", 1000) + platform(pre, "p-lost", 50) + idx(pre, "i-pre", "p-pre") + return v +} + +func mirrorRecords() *Records { + const up = "artifact-store://upstream/docker.io/library/golang@" + return &Records{References: []Record{ + {Reference: up + d("i-old"), State: "eligible"}, + {Reference: up + d("i-new"), State: "kept", Why: []string{"stood-on"}}, + }} +} + +func stateByName(entries []Entry) map[string]Entry { + out := map[string]Entry{} + for _, e := range entries { + out[e.Repository+" "+e.Digest] = e + } + return out +} + +func TestAPlatformManifestTakesTheStateOfTheIndexNamingIt(t *testing.T) { + v := mirrorWorld() + recs := mirrorRecords() + for i := range recs.References { + fill(&recs.References[i]) + } + got := stateByName(Classify(v, recs)) + up := "upstream/docker.io/library/golang " + for name, want := range map[string]string{ + up + d("p-amd"): StateEligible, // only the eligible index names it + up + d("p-arm"): StateKept, // both do: the more keeping wins + up + d("p-new"): StateKept, + "route-proxy/on-go_base " + d("p-pre"): StateUnrecorded, // its index has no record + "route-proxy/on-go_base " + d("p-lost"): StateUnrecorded, + } { + if got[name].State != want { + t.Errorf("%s reads %s, want %s", name, got[name].State, want) + } + } + if got[up+d("p-arm")].Index != d("i-new") { + t.Errorf("p-arm names its index as %q", got[up+d("p-arm")].Index) + } + // Only the top of an unrecorded copy is offered to the mirrors verb: its index, and a platform + // manifest no index names. + var mirrors []string + for _, e := range got { + if e.Mirror { + mirrors = append(mirrors, e.Digest) + } + } + if len(mirrors) != 2 || !got["route-proxy/on-go_base "+d("i-pre")].Mirror || !got["route-proxy/on-go_base "+d("p-lost")].Mirror { + t.Errorf("offered as unrecorded copies: %v", mirrors) + } + um := UnrecordedMirrors(v, Classify(v, recs)) + if len(um) != 2 || um[0].Reference != "artifact-store://route-proxy/on-go_base@"+d("i-pre") || um[0].OnlyTheir != 1+1+1000 { + t.Errorf("unrecorded copies: %+v", um) + } +} + +func TestLettingAnIndexGoTakesItsOwnPlatformsAndSaysWhatStaysAndNamesOne(t *testing.T) { + v := mirrorWorld() + recs := mirrorRecords() + for i := range recs.References { + fill(&recs.References[i]) + } + entries := Classify(v, recs) + old := "artifact-store://upstream/docker.io/library/golang@" + d("i-old") + gone := LetGoKeys(v, []string{old}, Spared(v, entries)) + up := "upstream/docker.io/library/golang" + want := map[string]bool{Key(up, d("i-old")): true, Key(up, d("p-amd")): true} + if !reflect.DeepEqual(gone, want) { + t.Fatalf("would go: %v; want the old index and the platform only it names", gone) + } + if o := Orphaned(v, gone); len(o) != 0 { + t.Fatalf("what stays names something that goes: %v", o) + } + // Spared nothing, the shared platform goes too, and the kept index that names it is said. + all := LetGoKeys(v, []string{old}, nil) + if o := Orphaned(v, all); len(o) != 1 || o[0] != Key(up, d("i-new")) { + t.Fatalf("a kept index losing a platform was not said: %v", o) + } + out := Collected(v, &CollectAnswer{DryRun: true, WouldLetGo: []string{old}, Eligible: 1}, true, entries) + if out["manifests_gone"] != 2 { + t.Fatalf("collect said %v manifests go", out["manifests_gone"]) + } + if out["collector_frees_after_bytes"].(int64)-out["collector_frees_now_bytes"].(int64) != 1+1+100 { + t.Fatalf("collect frees %v more, want the index, its own platform and that platform's layer", + out["collector_frees_after_bytes"].(int64)-out["collector_frees_now_bytes"].(int64)) + } +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index ad9adb72..df98e9c8 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -256,7 +256,7 @@ func TestBytesFreedCountASharedBlobAsKept(t *testing.T) { v := view(t, world()) a := &CollectAnswer{DryRun: true, Eligible: 2, WouldLetGo: []string{ "artifact-store://app/server@" + d("m-old"), "artifact-store://app/tools/blobs/" + d("a-old")}} - out := Collected(v, a, true) + out := Collected(v, a, true, nil) now, after := out["collector_frees_now_bytes"].(int64), out["collector_frees_after_bytes"].(int64) // m-old's own content, old-only and a-old and its holder go; shared and cfg stay with m-kept. want := int64(7000) + 500 + 4000 + v.L.Blobs[d("m-old")] + v.L.Blobs[d("h-old")] diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go b/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go index 2e03b143..df0600e1 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go @@ -55,9 +55,11 @@ func Tools(s *Store, c Controller) []stdio.Tool { Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " + "names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " + "eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " + - "a named document the controller keeps under a tag, or unrecorded — no record names it. Counted and sized by " + + "a named document the controller keeps under a tag, or unrecorded — no record names it. A platform manifest " + + "takes the state of the recorded index naming it. Counted and sized by " + "state and by repository; each manifest listed when one repository is asked. Then what the records keep that " + - "the store does not hold. Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)", + "the store does not hold, and the unrecorded copies of upstream images the controller's mirrors verb may " + + "record (novox/hq ADR 0257). Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)", Input: map[string]any{"repository": repoArg}, Run: func(args map[string]any) (any, error) { repo := text(args, "repository") @@ -108,7 +110,13 @@ func Tools(s *Store, c Controller) []stdio.Tool { if err != nil { return map[string]any{"controller": answer, "bytes_not_worked_out": err.Error()}, nil } - return Collected(v, answer, dry), nil + // What a kept index names stays when an index goes (novox/hq ADR 0257): read from the + // records, as the controller's sweep reads it. + recs, err := c.Artifacts("") + if err != nil { + return map[string]any{"controller": answer, "bytes_not_worked_out": "the records could not be read: " + err.Error()}, nil + } + return Collected(v, answer, dry, Classify(v, recs)), nil }, }, } @@ -244,10 +252,20 @@ func References(v *View, recs *Records, only string) (map[string]any, error) { if len(missing) > 0 { said = append(said, fmt.Sprintf("%d references the records keep are not in the store", len(missing))) } + mirrors := UnrecordedMirrors(v, entries) + if len(mirrors) > 0 { + var only int64 + for _, m := range mirrors { + only += m.OnlyTheir + } + said = append(said, fmt.Sprintf("%d unrecorded copies of upstream images, %s marked by them alone: the "+ + "controller's mirrors verb may record them as the mesh's (novox/hq ADR 0257), listed in unrecorded_mirrors", + len(mirrors), human(only))) + } out := map[string]any{ "store": recs.Store, "kept_builds": recs.KeptBuilds, "records": recs.Counts, "states": whole, "repositories": repos, "missing": nonNil(missing), "said": said, - "removable": collectionRemovesThese, + "removable": collectionRemovesThese, "unrecorded_mirrors": mirrorsOrEmpty(mirrors), } if only != "" { out["manifests"] = byRepo[only] @@ -262,12 +280,13 @@ func References(v *View, recs *Records, only string) (map[string]any, error) { } // Collected answers artifact_store_collect: the controller's answer, and what the store's collector frees. -func Collected(v *View, a *CollectAnswer, dry bool) map[string]any { +func Collected(v *View, a *CollectAnswer, dry bool, entries []Entry) map[string]any { refs := a.LetGo if dry { refs = a.WouldLetGo } - gone := LetGoKeys(v, refs) + gone := LetGoKeys(v, refs, Spared(v, entries)) + orphaned := Orphaned(v, gone) now, nowBlobs := v.Unmarked(nil) after, afterBlobs := v.Unmarked(gone) said := []string{} @@ -282,6 +301,10 @@ func Collected(v *View, a *CollectAnswer, dry bool) map[string]any { said = append(said, fmt.Sprintf("the nightly collector frees %s tonight as the store is now", human(now)), fmt.Sprintf("and %s once those %d manifests are gone (%s more)", human(after), len(gone), human(after-now))) + if len(orphaned) > 0 { + said = append(said, fmt.Sprintf("%d manifests that stay name a platform manifest this removes, and would "+ + "lose it: listed in platforms_named_by_what_stays — read them before a real run", len(orphaned))) + } if dry && a.Eligible > len(a.WouldLetGo) { said = append(said, fmt.Sprintf("the controller listed %d of %d eligible: the bytes cover only those listed", len(a.WouldLetGo), a.Eligible)) } @@ -289,7 +312,7 @@ func Collected(v *View, a *CollectAnswer, dry bool) map[string]any { "dry_run": dry, "controller": a, "collector_frees_now_bytes": now, "collector_frees_now_blobs": nowBlobs, "collector_frees_after_bytes": after, "collector_frees_after_blobs": afterBlobs, - "manifests_gone": len(gone), "said": said, + "manifests_gone": len(gone), "said": said, "platforms_named_by_what_stays": nonNil(orphaned), "note": "a manifest let go of frees no bytes until the store's nightly collector runs with the store held still (ADR 0189 §4)", } if u := unreadNote(v); u != nil { @@ -298,6 +321,13 @@ func Collected(v *View, a *CollectAnswer, dry bool) map[string]any { return out } +func mirrorsOrEmpty(m []UnrecordedMirror) []UnrecordedMirror { + if m == nil { + return []UnrecordedMirror{} + } + return m +} + func nonNil(s []string) []string { if s == nil { return []string{}