package main import ( "regexp" "sort" ) // The states a manifest the store holds can be in, as the records see it (novox/hq to-be 51). const ( StateKept = "kept" StateHolderKept = "holder-of-kept-archive" StateEligible = "eligible" StateHolderEligible = "holder-of-eligible-archive" StateCollectedPresent = "let-go-yet-present" StateNamedDocument = "named-document" StateUnrecorded = "unrecorded" collectionRemovesThese = "eligible and holder-of-eligible-archive, through the controller's collect; nothing else" ) // Entry is one manifest the store holds, and what the records say of it. type Entry struct { Repository string `json:"repository"` Digest string `json:"digest"` State string `json:"state"` Why []string `json:"why,omitempty"` // Archive is the archive a holder keeps. Archive string `json:"archive,omitempty"` Tags []string `json:"tags,omitempty"` // Bytes are the blobs it marks, its own content included. Bytes int64 `json:"bytes"` // Unread says its content could not be read, so what it marks is not known beyond itself. Unread string `json:"unread,omitempty"` // Index is the index of the same repository whose state a platform manifest takes (novox/hq ADR // 0257): the controller lets an index go with the platform manifests it names, so they are its. Index string `json:"index,omitempty"` // Mirror says it is an unrecorded copy at the top of a repository the mirror writes — what the // controller's mirrors verb may record as the mesh's (novox/hq ADR 0257). Mirror bool `json:"mirror,omitempty"` } // mirrorRepository is a repository only the build's mirror writes: one upstream image's repository, or a // module's own repository of a base from before ADR 0257. The controller's mirrors verb takes the same. var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`) // keeping orders the states a platform manifest may take from the indexes naming it: the most keeping // wins, so a platform two indexes name is as kept as the more kept of them. var keeping = map[string]int{ StateKept: 0, StateNamedDocument: 1, StateHolderKept: 2, StateEligible: 3, StateHolderEligible: 4, StateCollectedPresent: 5, } // recordIndex finds a record by repository and digest. type recordIndex map[string]*Record func indexRecords(r *Records) recordIndex { idx := recordIndex{} for i := range r.References { rec := &r.References[i] if rec.Repository == "" || rec.Digest == "" { continue } idx[rec.Kind+" "+Key(rec.Repository, rec.Digest)] = rec } return idx } // Classify gives every manifest the store holds one state. func Classify(v *View, recs *Records) []Entry { idx := indexRecords(recs) var out []Entry for _, name := range v.L.RepoNames() { r := v.L.Repos[name] tagsAt := map[string][]string{} for t, d := range r.Tags { tagsAt[d] = append(tagsAt[d], t) } digests := make([]string, 0, len(r.Revisions)) for d := range r.Revisions { digests = append(digests, d) } sort.Strings(digests) for _, d := range digests { e := Entry{Repository: name, Digest: d, State: StateUnrecorded, Tags: tagsAt[d]} sort.Strings(e.Tags) blobs := map[string]bool{} for _, b := range v.Marks(name, d) { blobs[b] = true } e.Bytes = v.Bytes(blobs) e.Unread = v.Unread[Key(name, d)] m := v.M[Key(name, d)] if rec := idx["image "+Key(name, d)]; rec != nil { e.State, e.Why = stateOf(rec, false), rec.Why } else if m.Holder() { archive := m.Layers[0] if rec := idx["archive "+Key(name, archive)]; rec != nil { e.State, e.Why, e.Archive = stateOf(rec, true), rec.Why, archive } else if len(e.Tags) > 0 { e.State = StateNamedDocument } } out = append(out, e) } } return platformsOf(v, out) } // platformsOf gives each unrecorded platform manifest the state of the recorded index of its repository // that names it, the most keeping when several do, and marks each unrecorded copy at the top of a mirror // repository. An index no record names leaves its platform manifests unrecorded with it. func platformsOf(v *View, entries []Entry) []Entry { at := map[string]int{} for i, e := range entries { at[Key(e.Repository, e.Digest)] = i } named := map[string]bool{} for _, e := range entries { m := v.M[Key(e.Repository, e.Digest)] if m == nil { continue } for _, c := range m.Children { named[Key(e.Repository, c)] = true } } for _, e := range entries { rank, recorded := keeping[e.State] if !recorded { continue } m := v.M[Key(e.Repository, e.Digest)] if m == nil { continue } for _, c := range m.Children { i, held := at[Key(e.Repository, c)] if !held { continue } child := &entries[i] if current, was := keeping[child.State]; child.State != StateUnrecorded && (!was || current <= rank) { continue } child.State, child.Why, child.Index = e.State, e.Why, e.Digest } } for i := range entries { e := &entries[i] if e.State == StateUnrecorded && !named[Key(e.Repository, e.Digest)] && mirrorRepository.MatchString(e.Repository) { e.Mirror = true } } return entries } func stateOf(rec *Record, holder bool) string { switch rec.State { case "kept": if holder { return StateHolderKept } return StateKept case "eligible": if holder { return StateHolderEligible } return StateEligible case "collected": return StateCollectedPresent } // A state this bundle does not know is kept: never offered as removable. return StateKept } // Missing are the references the records keep that the store does not hold: an image whose manifest // is not in its repository, an archive whose blob is not in the store. func Missing(v *View, recs *Records) []string { var out []string for _, rec := range recs.References { if rec.State != "kept" || rec.Repository == "" || rec.Digest == "" { continue } switch rec.Kind { case "image": if r := v.L.Repos[rec.Repository]; r == nil || !r.Revisions[rec.Digest] { out = append(out, rec.Reference) } case "archive": if _, ok := v.L.Blobs[rec.Digest]; !ok { out = append(out, rec.Reference) } } } sort.Strings(out) return out } // StateSum is the manifests of one state: how many, the bytes they mark together, and the bytes only // they mark — what the store would give back if they alone went. type StateSum struct { Manifests int `json:"manifests"` Bytes int64 `json:"bytes"` OnlyTheirBytes int64 `json:"only_their_bytes"` } // Summarise sums entries by state, over the whole store or one repository's entries, the "only theirs" // always judged against every manifest in the store. func Summarise(v *View, entries []Entry, all []Entry) map[string]*StateSum { statesOf := map[string]map[string]bool{} for _, e := range all { for _, b := range v.Marks(e.Repository, e.Digest) { if statesOf[b] == nil { statesOf[b] = map[string]bool{} } statesOf[b][e.State] = true } } sums := map[string]*StateSum{} blobs := map[string]map[string]bool{} for _, e := range entries { s := sums[e.State] if s == nil { s = &StateSum{} sums[e.State] = s blobs[e.State] = map[string]bool{} } s.Manifests++ for _, b := range v.Marks(e.Repository, e.Digest) { blobs[e.State][b] = true } } for state, set := range blobs { sums[state].Bytes = v.Bytes(set) only := map[string]bool{} for b := range set { if len(statesOf[b]) == 1 { only[b] = true } } sums[state].OnlyTheirBytes = v.Bytes(only) } return sums } // LetGoKeys are the manifests the store would no longer hold once the controller lets go of these // references: an image's manifest and, for an index, the platform manifests it names that no kept index // of the repository names too (novox/hq ADR 0257); and an archive's holder. Spared is the manifests a // kept index names, by Key. func LetGoKeys(v *View, refs []string, spared map[string]bool) map[string]bool { out := map[string]bool{} for _, ref := range refs { repo, digest, kind := splitReference(ref) r := v.L.Repos[repo] if r == nil { continue } switch kind { case "image": if r.Revisions[digest] { out[Key(repo, digest)] = true if m := v.M[Key(repo, digest)]; m != nil { for _, c := range m.Children { if r.Revisions[c] && !spared[Key(repo, c)] { out[Key(repo, c)] = true } } } } case "archive": for d := range r.Revisions { if m := v.M[Key(repo, d)]; m.Holder() && m.Layers[0] == digest { out[Key(repo, d)] = true } } } } return out } // Spared is every manifest the controller's sweep leaves in place when it lets an index go: each kept // manifest, and every platform manifest a kept index names (novox/hq ADR 0257). func Spared(v *View, entries []Entry) map[string]bool { out := map[string]bool{} for _, e := range entries { if e.State != StateKept || e.Index != "" { continue } out[Key(e.Repository, e.Digest)] = true if m := v.M[Key(e.Repository, e.Digest)]; m != nil { for _, c := range m.Children { out[Key(e.Repository, c)] = true } } } return out } // Orphaned are the manifests that stay and name a platform manifest collection would remove: what the // controller cannot see, because it reads only the records, said before anybody runs it for real. func Orphaned(v *View, gone map[string]bool) []string { var out []string for _, name := range v.L.RepoNames() { for d := range v.L.Repos[name].Revisions { if gone[Key(name, d)] { continue } if m := v.M[Key(name, d)]; m != nil { for _, c := range m.Children { if gone[Key(name, c)] { out = append(out, Key(name, d)) break } } } } } sort.Strings(out) return out } // UnrecordedMirror is one unrecorded copy at the top of a mirror repository, with what it and its // platform manifests mark that nothing else does. type UnrecordedMirror struct { Reference string `json:"reference"` Bytes int64 `json:"bytes"` OnlyTheir int64 `json:"only_their_bytes"` } // UnrecordedMirrors are the copies the controller's mirrors verb may record, largest first. func UnrecordedMirrors(v *View, entries []Entry) []UnrecordedMirror { marks := map[string]int{} for name, r := range v.L.Repos { for d := range r.Revisions { for _, b := range v.Marks(name, d) { marks[b]++ } } } var out []UnrecordedMirror for _, e := range entries { if !e.Mirror { continue } family := map[string]bool{} own := map[string]int{} add := func(digest string) { for _, b := range v.Marks(e.Repository, digest) { family[b] = true own[b]++ } } add(e.Digest) if m := v.M[Key(e.Repository, e.Digest)]; m != nil { for _, c := range m.Children { add(c) } } only := map[string]bool{} for b, n := range own { if marks[b] == n { only[b] = true } } out = append(out, UnrecordedMirror{Reference: "artifact-store://" + e.Repository + "@" + e.Digest, Bytes: v.Bytes(family), OnlyTheir: v.Bytes(only)}) } sort.Slice(out, func(i, j int) bool { if out[i].OnlyTheir != out[j].OnlyTheir { return out[i].OnlyTheir > out[j].OnlyTheir } return out[i].Reference < out[j].Reference }) return out }