mesh-catalog/modules/artifact-store-tools/cmd/artifact-store-tools/classify.go

package main

import (
	"regexp"
	"sort"
)

// The states a manifest the store holds can be in, as the records see it (novox/hq to-be 51).
const (
	StateKept             = "kept"
	StateHolderKept       = "holder-of-kept-archive"
	StateEligible         = "eligible"
	StateHolderEligible   = "holder-of-eligible-archive"
	StateCollectedPresent = "let-go-yet-present"
	StateNamedDocument    = "named-document"
	// StateNamedByDefinition is a manifest no record names that a definition pins by digest, under the
	// store's current name or an older one (novox/hq issue 323): a machine may run it, so it is kept.
	StateNamedByDefinition = "named-by-a-definition"
	StateUnrecorded        = "unrecorded"
	collectionRemovesThese = "eligible and holder-of-eligible-archive, through the controller's collect; nothing else"
)

// Entry is one manifest the store holds, and what the records say of it.
type Entry struct {
	Repository string   `json:"repository"`
	Digest     string   `json:"digest"`
	State      string   `json:"state"`
	Why        []string `json:"why,omitempty"`
	// Archive is the archive a holder keeps.
	Archive string   `json:"archive,omitempty"`
	Tags    []string `json:"tags,omitempty"`
	// Bytes are the blobs it marks, its own content included.
	Bytes int64 `json:"bytes"`
	// Unread says its content could not be read, so what it marks is not known beyond itself.
	Unread string `json:"unread,omitempty"`
	// Index is the index of the same repository whose state a platform manifest takes (novox/hq ADR
	// 0257): the controller lets an index go with the platform manifests it names, so they are its.
	Index string `json:"index,omitempty"`
	// Mirror says it is an unrecorded copy at the top of a repository the mirror writes — what the
	// controller's mirrors verb may record as the mesh's (novox/hq ADR 0257).
	Mirror bool `json:"mirror,omitempty"`
}

// mirrorRepository is a repository only the build's mirror writes: one upstream image's repository, or a
// module's own repository of a base from before ADR 0257. The controller's mirrors verb takes the same.
var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z0-9][a-z0-9._-]*/on-[a-z0-9_]+)$`)

// keeping orders the states a platform manifest may take from the indexes naming it: the most keeping
// wins, so a platform two indexes name is as kept as the more kept of them.
var keeping = map[string]int{
	StateKept: 0, StateNamedByDefinition: 1, StateNamedDocument: 2, StateHolderKept: 3, StateEligible: 4,
	StateHolderEligible: 5, StateCollectedPresent: 6,
}

// recordIndex finds a record by repository and digest.
type recordIndex map[string]*Record

func indexRecords(r *Records) recordIndex {
	idx := recordIndex{}
	for i := range r.References {
		rec := &r.References[i]
		if rec.Repository == "" || rec.Digest == "" {
			continue
		}
		idx[rec.Kind+" "+Key(rec.Repository, rec.Digest)] = rec
	}
	return idx
}

// Classify gives every manifest the store holds one state. A manifest no record names that a definition
// pins is named by a definition; defs may be nil, when the definitions were not read.
func Classify(v *View, recs *Records, defs *Definitions) []Entry {
	idx := indexRecords(recs)
	var out []Entry
	for _, name := range v.L.RepoNames() {
		r := v.L.Repos[name]
		tagsAt := map[string][]string{}
		for t, d := range r.Tags {
			tagsAt[d] = append(tagsAt[d], t)
		}
		digests := make([]string, 0, len(r.Revisions))
		for d := range r.Revisions {
			digests = append(digests, d)
		}
		sort.Strings(digests)
		for _, d := range digests {
			e := Entry{Repository: name, Digest: d, State: StateUnrecorded, Tags: tagsAt[d]}
			sort.Strings(e.Tags)
			blobs := map[string]bool{}
			for _, b := range v.Marks(name, d) {
				blobs[b] = true
			}
			e.Bytes = v.Bytes(blobs)
			e.Unread = v.Unread[Key(name, d)]
			m := v.M[Key(name, d)]
			if rec := idx["image "+Key(name, d)]; rec != nil {
				e.State, e.Why = stateOf(rec, false), rec.Why
			} else if m.Holder() {
				archive := m.Layers[0]
				if rec := idx["archive "+Key(name, archive)]; rec != nil {
					e.State, e.Why, e.Archive = stateOf(rec, true), rec.Why, archive
				} else if len(e.Tags) > 0 {
					e.State = StateNamedDocument
				}
			}
			if e.State == StateUnrecorded && defs != nil {
				for _, p := range defs.Pins[Key(name, d)] {
					e.State = StateNamedByDefinition
					e.Why = append(e.Why, p.why())
				}
			}
			out = append(out, e)
		}
	}
	return platformsOf(v, out)
}

// platformsOf gives each unrecorded platform manifest the state of the recorded index of its repository
// that names it, the most keeping when several do, and marks each unrecorded copy at the top of a mirror
// repository. An index no record names leaves its platform manifests unrecorded with it.
func platformsOf(v *View, entries []Entry) []Entry {
	at := map[string]int{}
	for i, e := range entries {
		at[Key(e.Repository, e.Digest)] = i
	}
	named := map[string]bool{}
	for _, e := range entries {
		m := v.M[Key(e.Repository, e.Digest)]
		if m == nil {
			continue
		}
		for _, c := range m.Children {
			named[Key(e.Repository, c)] = true
		}
	}
	for _, e := range entries {
		rank, recorded := keeping[e.State]
		if !recorded {
			continue
		}
		m := v.M[Key(e.Repository, e.Digest)]
		if m == nil {
			continue
		}
		for _, c := range m.Children {
			i, held := at[Key(e.Repository, c)]
			if !held {
				continue
			}
			child := &entries[i]
			if current, was := keeping[child.State]; child.State != StateUnrecorded && (!was || current <= rank) {
				continue
			}
			child.State, child.Why, child.Index = e.State, e.Why, e.Digest
		}
	}
	for i := range entries {
		e := &entries[i]
		if e.State == StateUnrecorded && !named[Key(e.Repository, e.Digest)] && mirrorRepository.MatchString(e.Repository) {
			e.Mirror = true
		}
	}
	return entries
}

func stateOf(rec *Record, holder bool) string {
	switch rec.State {
	case "kept":
		if holder {
			return StateHolderKept
		}
		return StateKept
	case "eligible":
		if holder {
			return StateHolderEligible
		}
		return StateEligible
	case "collected":
		return StateCollectedPresent
	}
	// A state this bundle does not know is kept: never offered as removable.
	return StateKept
}

// Missing are the references the records keep that the store does not hold: an image whose manifest
// is not in its repository, an archive whose blob is not in the store.
func Missing(v *View, recs *Records) []string {
	var out []string
	for _, rec := range recs.References {
		if rec.State != "kept" || rec.Repository == "" || rec.Digest == "" {
			continue
		}
		switch rec.Kind {
		case "image":
			if r := v.L.Repos[rec.Repository]; r == nil || !r.Revisions[rec.Digest] {
				out = append(out, rec.Reference)
			}
		case "archive":
			if _, ok := v.L.Blobs[rec.Digest]; !ok {
				out = append(out, rec.Reference)
			}
		}
	}
	sort.Strings(out)
	return out
}

// StateSum is the manifests of one state: how many, the bytes they mark together, and the bytes only
// they mark — what the store would give back if they alone went.
type StateSum struct {
	Manifests      int   `json:"manifests"`
	Bytes          int64 `json:"bytes"`
	OnlyTheirBytes int64 `json:"only_their_bytes"`
}

// Summarise sums entries by state, over the whole store or one repository's entries, the "only theirs"
// always judged against every manifest in the store.
func Summarise(v *View, entries []Entry, all []Entry) map[string]*StateSum {
	statesOf := map[string]map[string]bool{}
	for _, e := range all {
		for _, b := range v.Marks(e.Repository, e.Digest) {
			if statesOf[b] == nil {
				statesOf[b] = map[string]bool{}
			}
			statesOf[b][e.State] = true
		}
	}
	sums := map[string]*StateSum{}
	blobs := map[string]map[string]bool{}
	for _, e := range entries {
		s := sums[e.State]
		if s == nil {
			s = &StateSum{}
			sums[e.State] = s
			blobs[e.State] = map[string]bool{}
		}
		s.Manifests++
		for _, b := range v.Marks(e.Repository, e.Digest) {
			blobs[e.State][b] = true
		}
	}
	for state, set := range blobs {
		sums[state].Bytes = v.Bytes(set)
		only := map[string]bool{}
		for b := range set {
			if len(statesOf[b]) == 1 {
				only[b] = true
			}
		}
		sums[state].OnlyTheirBytes = v.Bytes(only)
	}
	return sums
}

// LetGoKeys are the manifests the store would no longer hold once the controller lets go of these
// references: an image's manifest and, for an index, the platform manifests it names that no kept index
// of the repository names too (novox/hq ADR 0257); and an archive's holder. Spared is the manifests a
// kept index names, by Key.
func LetGoKeys(v *View, refs []string, spared map[string]bool) map[string]bool {
	out := map[string]bool{}
	for _, ref := range refs {
		repo, digest, kind := splitReference(ref)
		r := v.L.Repos[repo]
		if r == nil {
			continue
		}
		switch kind {
		case "image":
			if r.Revisions[digest] {
				out[Key(repo, digest)] = true
				if m := v.M[Key(repo, digest)]; m != nil {
					for _, c := range m.Children {
						if r.Revisions[c] && !spared[Key(repo, c)] {
							out[Key(repo, c)] = true
						}
					}
				}
			}
		case "archive":
			for d := range r.Revisions {
				if m := v.M[Key(repo, d)]; m.Holder() && m.Layers[0] == digest {
					out[Key(repo, d)] = true
				}
			}
		}
	}
	return out
}

// Spared is every manifest the controller's sweep leaves in place when it lets an index go: each kept
// manifest, and every platform manifest a kept index names (novox/hq ADR 0257).
func Spared(v *View, entries []Entry) map[string]bool {
	out := map[string]bool{}
	for _, e := range entries {
		if e.State != StateKept || e.Index != "" {
			continue
		}
		out[Key(e.Repository, e.Digest)] = true
		if m := v.M[Key(e.Repository, e.Digest)]; m != nil {
			for _, c := range m.Children {
				out[Key(e.Repository, c)] = true
			}
		}
	}
	return out
}

// Orphaned are the manifests that stay and name a platform manifest collection would remove: what the
// controller cannot see, because it reads only the records, said before anybody runs it for real.
func Orphaned(v *View, gone map[string]bool) []string {
	var out []string
	for _, name := range v.L.RepoNames() {
		for d := range v.L.Repos[name].Revisions {
			if gone[Key(name, d)] {
				continue
			}
			if m := v.M[Key(name, d)]; m != nil {
				for _, c := range m.Children {
					if gone[Key(name, c)] {
						out = append(out, Key(name, d))
						break
					}
				}
			}
		}
	}
	sort.Strings(out)
	return out
}

// UnrecordedMirror is one unrecorded copy at the top of a mirror repository, with what it and its
// platform manifests mark that nothing else does.
type UnrecordedMirror struct {
	Reference string `json:"reference"`
	Bytes     int64  `json:"bytes"`
	OnlyTheir int64  `json:"only_their_bytes"`
}

// UnrecordedMirrors are the copies the controller's mirrors verb may record, largest first.
func UnrecordedMirrors(v *View, entries []Entry) []UnrecordedMirror {
	marks := map[string]int{}
	for name, r := range v.L.Repos {
		for d := range r.Revisions {
			for _, b := range v.Marks(name, d) {
				marks[b]++
			}
		}
	}
	var out []UnrecordedMirror
	for _, e := range entries {
		if !e.Mirror {
			continue
		}
		family := map[string]bool{}
		own := map[string]int{}
		add := func(digest string) {
			for _, b := range v.Marks(e.Repository, digest) {
				family[b] = true
				own[b]++
			}
		}
		add(e.Digest)
		if m := v.M[Key(e.Repository, e.Digest)]; m != nil {
			for _, c := range m.Children {
				add(c)
			}
		}
		only := map[string]bool{}
		for b, n := range own {
			if marks[b] == n {
				only[b] = true
			}
		}
		out = append(out, UnrecordedMirror{Reference: "artifact-store://" + e.Repository + "@" + e.Digest,
			Bytes: v.Bytes(family), OnlyTheir: v.Bytes(only)})
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].OnlyTheir != out[j].OnlyTheir {
			return out[i].OnlyTheir > out[j].OnlyTheir
		}
		return out[i].Reference < out[j].Reference
	})
	return out
}