jochen
2dffc48c81
The store's tools counted each platform of an upstream image apart from its index, so a kept copy read as unrecorded and a let-go index's bytes were never counted as freed. The copies no record names are now listed for the controller's mirrors verb, and a dry run says what stays yet names a platform collection would take (hq ADR 0257, issue 321).
382 lines
14 KiB
Go
382 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
|
|
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
|
|
func Tools(s *Store, c Controller) []stdio.Tool {
|
|
ctx := context.Background()
|
|
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
|
|
return []stdio.Tool{
|
|
{
|
|
Name: "artifact_store_repositories",
|
|
Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " +
|
|
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " +
|
|
"A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " +
|
|
"the store's own files; changes nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
|
Input: map[string]any{"repository": repoArg},
|
|
Run: func(args map[string]any) (any, error) {
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Repositories(v, text(args, "repository"))
|
|
},
|
|
},
|
|
{
|
|
Name: "artifact_store_usage",
|
|
Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " +
|
|
"more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " +
|
|
"frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)",
|
|
Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
top, err := count(args, "top", 10, 1, 200)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Usage(v, top), nil
|
|
},
|
|
},
|
|
{
|
|
Name: "artifact_store_references",
|
|
Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " +
|
|
"names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " +
|
|
"eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " +
|
|
"a named document the controller keeps under a tag, or unrecorded — no record names it. A platform manifest " +
|
|
"takes the state of the recorded index naming it. Counted and sized by " +
|
|
"state and by repository; each manifest listed when one repository is asked. Then what the records keep that " +
|
|
"the store does not hold, and the unrecorded copies of upstream images the controller's mirrors verb may " +
|
|
"record (novox/hq ADR 0257). Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)",
|
|
Input: map[string]any{"repository": repoArg},
|
|
Run: func(args map[string]any) (any, error) {
|
|
repo := text(args, "repository")
|
|
recs, err := c.Artifacts("")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return References(v, recs, repo)
|
|
},
|
|
},
|
|
{
|
|
Name: "artifact_store_collect",
|
|
Description: "Collect what the mesh made and keeps for no reason (novox/hq ADR 0189, ADR 0251 §3). A dry run unless " +
|
|
"dry_run is false: it asks the controller's collect what it would let go of, and works out how many bytes the " +
|
|
"store's nightly collector would free once those are gone, beside what it frees tonight anyway. A real run needs " +
|
|
"why: it asks the controller's collect to hold every kept archive and let go of the eligible ones, recorded as a " +
|
|
"hand-act with the why; the bytes come back at the nightly collection. Never removes an unrecorded manifest, and " +
|
|
"never deletes through the store's door itself. (a)",
|
|
Input: map[string]any{
|
|
"dry_run": map[string]any{"type": "boolean", "description": "true (the default): say what would go, change nothing"},
|
|
"why": map[string]any{"type": "string", "description": "why — required for a real run, and recorded as the hand-act's reason"},
|
|
"most": map[string]any{"type": "integer", "description": "the most artifacts one real run lets go of (the controller's default when absent)"},
|
|
},
|
|
Run: func(args map[string]any) (any, error) {
|
|
dry := true
|
|
if b, given := args["dry_run"].(bool); given {
|
|
dry = b
|
|
} else if t := text(args, "dry_run"); t == "false" {
|
|
dry = false
|
|
}
|
|
why := text(args, "why")
|
|
if !dry && why == "" {
|
|
return nil, fmt.Errorf("a real collection needs why: nothing was asked of the controller")
|
|
}
|
|
most, err := count(args, "most", 0, 1, 5000)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
answer, err := c.Collect(why, !dry, most)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return map[string]any{"controller": answer, "bytes_not_worked_out": err.Error()}, nil
|
|
}
|
|
// What a kept index names stays when an index goes (novox/hq ADR 0257): read from the
|
|
// records, as the controller's sweep reads it.
|
|
recs, err := c.Artifacts("")
|
|
if err != nil {
|
|
return map[string]any{"controller": answer, "bytes_not_worked_out": "the records could not be read: " + err.Error()}, nil
|
|
}
|
|
return Collected(v, answer, dry, Classify(v, recs)), nil
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// View reads the store now.
|
|
func (s *Store) View(ctx context.Context) (*View, error) {
|
|
l, err := s.List(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
m, unread := s.Manifests(ctx, l)
|
|
return &View{L: l, M: m, Unread: unread}, nil
|
|
}
|
|
|
|
// unreadNote says what manifests that could not be read do to the numbers, when there are any.
|
|
func unreadNote(v *View) map[string]any {
|
|
if len(v.Unread) == 0 {
|
|
return nil
|
|
}
|
|
keys := make([]string, 0, len(v.Unread))
|
|
for k := range v.Unread {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
first := keys[0] + ": " + v.Unread[keys[0]]
|
|
return map[string]any{
|
|
"manifests": len(keys),
|
|
"first": first,
|
|
"means": "what these mark beyond themselves is not known, so sizes may be low and the bytes said to be " +
|
|
"freed may be high; ask again — what was read is kept, so the next call reads only the rest",
|
|
}
|
|
}
|
|
|
|
// Repositories answers artifact_store_repositories.
|
|
func Repositories(v *View, only string) (map[string]any, error) {
|
|
shared := v.Shared()
|
|
var repos []map[string]any
|
|
for _, name := range v.L.RepoNames() {
|
|
if only != "" && name != only {
|
|
continue
|
|
}
|
|
r := v.L.Repos[name]
|
|
blobs := v.RepoBlobs(name)
|
|
sharedHere := map[string]bool{}
|
|
for b := range blobs {
|
|
if shared[b] {
|
|
sharedHere[b] = true
|
|
}
|
|
}
|
|
repos = append(repos, map[string]any{
|
|
"repository": name, "tags": tagsOf(r), "manifests": len(r.Revisions),
|
|
"bytes": v.Bytes(blobs), "size": human(v.Bytes(blobs)), "shared_bytes": v.Bytes(sharedHere),
|
|
})
|
|
}
|
|
if only != "" && len(repos) == 0 {
|
|
return nil, fmt.Errorf("the store holds no repository %q", only)
|
|
}
|
|
out := map[string]any{"count": len(repos), "repositories": repos,
|
|
"note": "a repository's size is the blobs its manifests mark; a blob shared with another repository is counted in each (shared_bytes)"}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Usage answers artifact_store_usage.
|
|
func Usage(v *View, top int) map[string]any {
|
|
type sized struct {
|
|
name string
|
|
bytes int64
|
|
n int
|
|
}
|
|
var all []sized
|
|
manifests := 0
|
|
for _, name := range v.L.RepoNames() {
|
|
n := len(v.L.Repos[name].Revisions)
|
|
manifests += n
|
|
all = append(all, sized{name, v.Bytes(v.RepoBlobs(name)), n})
|
|
}
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].bytes > all[j].bytes })
|
|
var largest []map[string]any
|
|
for i, s := range all {
|
|
if i >= top {
|
|
break
|
|
}
|
|
largest = append(largest, map[string]any{"repository": s.name, "bytes": s.bytes, "size": human(s.bytes), "manifests": s.n})
|
|
}
|
|
total := v.Total()
|
|
sharedBytes := v.Bytes(v.Shared())
|
|
freed, freedBlobs := v.Unmarked(nil)
|
|
out := map[string]any{
|
|
"total_bytes": total, "total": human(total), "blobs": len(v.L.Blobs), "manifests": manifests,
|
|
"repositories": len(v.L.Repos), "largest": largest,
|
|
"shared_bytes": sharedBytes, "collector_frees_next_bytes": freed, "collector_frees_next_blobs": freedBlobs,
|
|
"said": []string{
|
|
fmt.Sprintf("the store holds %s in %d blobs, %d manifests in %d repositories", human(total), len(v.L.Blobs), manifests, len(v.L.Repos)),
|
|
fmt.Sprintf("%s is marked by more than one repository's manifests", human(sharedBytes)),
|
|
fmt.Sprintf("the nightly collector frees %s in %d blobs no manifest marks", human(freed), freedBlobs),
|
|
},
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out
|
|
}
|
|
|
|
// References answers artifact_store_references.
|
|
func References(v *View, recs *Records, only string) (map[string]any, error) {
|
|
if only != "" && v.L.Repos[only] == nil {
|
|
return nil, fmt.Errorf("the store holds no repository %q", only)
|
|
}
|
|
entries := Classify(v, recs)
|
|
byRepo := map[string][]Entry{}
|
|
for _, e := range entries {
|
|
byRepo[e.Repository] = append(byRepo[e.Repository], e)
|
|
}
|
|
var repos []map[string]any
|
|
for _, name := range v.L.RepoNames() {
|
|
if only != "" && name != only {
|
|
continue
|
|
}
|
|
repos = append(repos, map[string]any{"repository": name, "states": Summarise(v, byRepo[name], entries)})
|
|
}
|
|
whole := Summarise(v, entries, entries)
|
|
said := []string{}
|
|
for _, state := range []string{StateKept, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
|
if s := whole[state]; s != nil {
|
|
said = append(said, fmt.Sprintf("%s: %d manifests marking %s, %s of it marked by nothing else", state, s.Manifests, human(s.Bytes), human(s.OnlyTheirBytes)))
|
|
}
|
|
}
|
|
missing := Missing(v, recs)
|
|
if len(missing) > 0 {
|
|
said = append(said, fmt.Sprintf("%d references the records keep are not in the store", len(missing)))
|
|
}
|
|
mirrors := UnrecordedMirrors(v, entries)
|
|
if len(mirrors) > 0 {
|
|
var only int64
|
|
for _, m := range mirrors {
|
|
only += m.OnlyTheir
|
|
}
|
|
said = append(said, fmt.Sprintf("%d unrecorded copies of upstream images, %s marked by them alone: the "+
|
|
"controller's mirrors verb may record them as the mesh's (novox/hq ADR 0257), listed in unrecorded_mirrors",
|
|
len(mirrors), human(only)))
|
|
}
|
|
out := map[string]any{
|
|
"store": recs.Store, "kept_builds": recs.KeptBuilds, "records": recs.Counts,
|
|
"states": whole, "repositories": repos, "missing": nonNil(missing), "said": said,
|
|
"removable": collectionRemovesThese, "unrecorded_mirrors": mirrorsOrEmpty(mirrors),
|
|
}
|
|
if only != "" {
|
|
out["manifests"] = byRepo[only]
|
|
}
|
|
if recs.Note != "" {
|
|
out["note"] = recs.Note
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Collected answers artifact_store_collect: the controller's answer, and what the store's collector frees.
|
|
func Collected(v *View, a *CollectAnswer, dry bool, entries []Entry) map[string]any {
|
|
refs := a.LetGo
|
|
if dry {
|
|
refs = a.WouldLetGo
|
|
}
|
|
gone := LetGoKeys(v, refs, Spared(v, entries))
|
|
orphaned := Orphaned(v, gone)
|
|
now, nowBlobs := v.Unmarked(nil)
|
|
after, afterBlobs := v.Unmarked(gone)
|
|
said := []string{}
|
|
if dry {
|
|
said = append(said, fmt.Sprintf("dry run: the controller would let go of %d of %d eligible artifacts; nothing was changed", len(a.WouldLetGo), a.Eligible))
|
|
} else {
|
|
said = append(said, fmt.Sprintf("the controller let go of %d artifacts (%d left, %d skipped)", len(a.LetGo), a.Left, a.Skipped))
|
|
if a.Stopped != "" {
|
|
said = append(said, "it stopped: "+a.Stopped)
|
|
}
|
|
}
|
|
said = append(said,
|
|
fmt.Sprintf("the nightly collector frees %s tonight as the store is now", human(now)),
|
|
fmt.Sprintf("and %s once those %d manifests are gone (%s more)", human(after), len(gone), human(after-now)))
|
|
if len(orphaned) > 0 {
|
|
said = append(said, fmt.Sprintf("%d manifests that stay name a platform manifest this removes, and would "+
|
|
"lose it: listed in platforms_named_by_what_stays — read them before a real run", len(orphaned)))
|
|
}
|
|
if dry && a.Eligible > len(a.WouldLetGo) {
|
|
said = append(said, fmt.Sprintf("the controller listed %d of %d eligible: the bytes cover only those listed", len(a.WouldLetGo), a.Eligible))
|
|
}
|
|
out := map[string]any{
|
|
"dry_run": dry, "controller": a,
|
|
"collector_frees_now_bytes": now, "collector_frees_now_blobs": nowBlobs,
|
|
"collector_frees_after_bytes": after, "collector_frees_after_blobs": afterBlobs,
|
|
"manifests_gone": len(gone), "said": said, "platforms_named_by_what_stays": nonNil(orphaned),
|
|
"note": "a manifest let go of frees no bytes until the store's nightly collector runs with the store held still (ADR 0189 §4)",
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mirrorsOrEmpty(m []UnrecordedMirror) []UnrecordedMirror {
|
|
if m == nil {
|
|
return []UnrecordedMirror{}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func nonNil(s []string) []string {
|
|
if s == nil {
|
|
return []string{}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func human(n int64) string {
|
|
switch {
|
|
case n >= 1<<30:
|
|
return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30))
|
|
case n >= 1<<20:
|
|
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
|
|
case n >= 1<<10:
|
|
return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
|
|
}
|
|
return fmt.Sprintf("%d B", n)
|
|
}
|
|
|
|
func text(args map[string]any, key string) string {
|
|
s, _ := args[key].(string)
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
// count is a whole-number argument, defaulted, refused below least, held at most.
|
|
func count(args map[string]any, key string, fallback, least, most int) (int, error) {
|
|
v, given := args[key]
|
|
if !given || v == nil {
|
|
return fallback, nil
|
|
}
|
|
var n int
|
|
switch x := v.(type) {
|
|
case float64:
|
|
if x != math.Trunc(x) {
|
|
return 0, fmt.Errorf("%s must be a whole number, not %v", key, x)
|
|
}
|
|
n = int(x)
|
|
case string:
|
|
i, err := strconv.Atoi(strings.TrimSpace(x))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("%s must be a whole number, not %q", key, x)
|
|
}
|
|
n = i
|
|
default:
|
|
return 0, fmt.Errorf("%s must be a whole number", key)
|
|
}
|
|
if n < least {
|
|
return 0, fmt.Errorf("%s must be at least %d", key, least)
|
|
}
|
|
return min(n, most), nil
|
|
}
|