jochen
434b52614d
The store had no working tools: its TypeScript client was never built, and nothing could count what the store holds that no record names. A Go bundle lists the store's files through its own container, reads each manifest through its door, and sets that beside the controller's records. Collection is asked of the controller, which decides and records; the store's tools never delete. The image.pushed event was declared and never emitted, and nothing consumes it, so it goes.
352 lines
12 KiB
Go
352 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// Tools are the store module's tools (novox/hq ADR 0251 §1). The first three only read; store_collect
|
|
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
|
|
func Tools(s *Store, c Controller) []stdio.Tool {
|
|
ctx := context.Background()
|
|
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
|
|
return []stdio.Tool{
|
|
{
|
|
Name: "store_repositories",
|
|
Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " +
|
|
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " +
|
|
"A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " +
|
|
"the store's own files and its door; changes nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
|
Input: map[string]any{"repository": repoArg},
|
|
Run: func(args map[string]any) (any, error) {
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Repositories(v, text(args, "repository"))
|
|
},
|
|
},
|
|
{
|
|
Name: "store_usage",
|
|
Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " +
|
|
"more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " +
|
|
"frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)",
|
|
Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
top, err := count(args, "top", 10, 1, 200)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Usage(v, top), nil
|
|
},
|
|
},
|
|
{
|
|
Name: "store_references",
|
|
Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " +
|
|
"names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " +
|
|
"eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " +
|
|
"a named document the controller keeps under a tag, or unrecorded — no record names it. Counted and sized by " +
|
|
"state and by repository; each manifest listed when one repository is asked. Then what the records keep that " +
|
|
"the store does not hold. Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)",
|
|
Input: map[string]any{"repository": repoArg},
|
|
Run: func(args map[string]any) (any, error) {
|
|
repo := text(args, "repository")
|
|
recs, err := c.Artifacts("")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return References(v, recs, repo)
|
|
},
|
|
},
|
|
{
|
|
Name: "store_collect",
|
|
Description: "Collect what the mesh made and keeps for no reason (novox/hq ADR 0189, ADR 0251 §3). A dry run unless " +
|
|
"dry_run is false: it asks the controller's collect what it would let go of, and works out how many bytes the " +
|
|
"store's nightly collector would free once those are gone, beside what it frees tonight anyway. A real run needs " +
|
|
"why: it asks the controller's collect to hold every kept archive and let go of the eligible ones, recorded as a " +
|
|
"hand-act with the why; the bytes come back at the nightly collection. Never removes an unrecorded manifest, and " +
|
|
"never deletes through the store's door itself. (a)",
|
|
Input: map[string]any{
|
|
"dry_run": map[string]any{"type": "boolean", "description": "true (the default): say what would go, change nothing"},
|
|
"why": map[string]any{"type": "string", "description": "why — required for a real run, and recorded as the hand-act's reason"},
|
|
"most": map[string]any{"type": "integer", "description": "the most artifacts one real run lets go of (the controller's default when absent)"},
|
|
},
|
|
Run: func(args map[string]any) (any, error) {
|
|
dry := true
|
|
if b, given := args["dry_run"].(bool); given {
|
|
dry = b
|
|
} else if t := text(args, "dry_run"); t == "false" {
|
|
dry = false
|
|
}
|
|
why := text(args, "why")
|
|
if !dry && why == "" {
|
|
return nil, fmt.Errorf("a real collection needs why: nothing was asked of the controller")
|
|
}
|
|
most, err := count(args, "most", 0, 1, 5000)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
answer, err := c.Collect(why, !dry, most)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return map[string]any{"controller": answer, "bytes_not_worked_out": err.Error()}, nil
|
|
}
|
|
return Collected(v, answer, dry), nil
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// View reads the store now.
|
|
func (s *Store) View(ctx context.Context) (*View, error) {
|
|
l, err := s.List(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
m, unread := s.Manifests(ctx, l)
|
|
return &View{L: l, M: m, Unread: unread}, nil
|
|
}
|
|
|
|
// unreadNote says what manifests that could not be read do to the numbers, when there are any.
|
|
func unreadNote(v *View) map[string]any {
|
|
if len(v.Unread) == 0 {
|
|
return nil
|
|
}
|
|
keys := make([]string, 0, len(v.Unread))
|
|
for k := range v.Unread {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
first := keys[0] + ": " + v.Unread[keys[0]]
|
|
return map[string]any{
|
|
"manifests": len(keys),
|
|
"first": first,
|
|
"means": "what these mark beyond themselves is not known, so sizes may be low and the bytes said to be " +
|
|
"freed may be high; ask again — what was read is kept, so the next call reads only the rest",
|
|
}
|
|
}
|
|
|
|
// Repositories answers store_repositories.
|
|
func Repositories(v *View, only string) (map[string]any, error) {
|
|
shared := v.Shared()
|
|
var repos []map[string]any
|
|
for _, name := range v.L.RepoNames() {
|
|
if only != "" && name != only {
|
|
continue
|
|
}
|
|
r := v.L.Repos[name]
|
|
blobs := v.RepoBlobs(name)
|
|
sharedHere := map[string]bool{}
|
|
for b := range blobs {
|
|
if shared[b] {
|
|
sharedHere[b] = true
|
|
}
|
|
}
|
|
repos = append(repos, map[string]any{
|
|
"repository": name, "tags": tagsOf(r), "manifests": len(r.Revisions),
|
|
"bytes": v.Bytes(blobs), "size": human(v.Bytes(blobs)), "shared_bytes": v.Bytes(sharedHere),
|
|
})
|
|
}
|
|
if only != "" && len(repos) == 0 {
|
|
return nil, fmt.Errorf("the store holds no repository %q", only)
|
|
}
|
|
out := map[string]any{"count": len(repos), "repositories": repos,
|
|
"note": "a repository's size is the blobs its manifests mark; a blob shared with another repository is counted in each (shared_bytes)"}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Usage answers store_usage.
|
|
func Usage(v *View, top int) map[string]any {
|
|
type sized struct {
|
|
name string
|
|
bytes int64
|
|
n int
|
|
}
|
|
var all []sized
|
|
manifests := 0
|
|
for _, name := range v.L.RepoNames() {
|
|
n := len(v.L.Repos[name].Revisions)
|
|
manifests += n
|
|
all = append(all, sized{name, v.Bytes(v.RepoBlobs(name)), n})
|
|
}
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].bytes > all[j].bytes })
|
|
var largest []map[string]any
|
|
for i, s := range all {
|
|
if i >= top {
|
|
break
|
|
}
|
|
largest = append(largest, map[string]any{"repository": s.name, "bytes": s.bytes, "size": human(s.bytes), "manifests": s.n})
|
|
}
|
|
total := v.Total()
|
|
sharedBytes := v.Bytes(v.Shared())
|
|
freed, freedBlobs := v.Unmarked(nil)
|
|
out := map[string]any{
|
|
"total_bytes": total, "total": human(total), "blobs": len(v.L.Blobs), "manifests": manifests,
|
|
"repositories": len(v.L.Repos), "largest": largest,
|
|
"shared_bytes": sharedBytes, "collector_frees_next_bytes": freed, "collector_frees_next_blobs": freedBlobs,
|
|
"said": []string{
|
|
fmt.Sprintf("the store holds %s in %d blobs, %d manifests in %d repositories", human(total), len(v.L.Blobs), manifests, len(v.L.Repos)),
|
|
fmt.Sprintf("%s is marked by more than one repository's manifests", human(sharedBytes)),
|
|
fmt.Sprintf("the nightly collector frees %s in %d blobs no manifest marks", human(freed), freedBlobs),
|
|
},
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out
|
|
}
|
|
|
|
// References answers store_references.
|
|
func References(v *View, recs *Records, only string) (map[string]any, error) {
|
|
if only != "" && v.L.Repos[only] == nil {
|
|
return nil, fmt.Errorf("the store holds no repository %q", only)
|
|
}
|
|
entries := Classify(v, recs)
|
|
byRepo := map[string][]Entry{}
|
|
for _, e := range entries {
|
|
byRepo[e.Repository] = append(byRepo[e.Repository], e)
|
|
}
|
|
var repos []map[string]any
|
|
for _, name := range v.L.RepoNames() {
|
|
if only != "" && name != only {
|
|
continue
|
|
}
|
|
repos = append(repos, map[string]any{"repository": name, "states": Summarise(v, byRepo[name], entries)})
|
|
}
|
|
whole := Summarise(v, entries, entries)
|
|
said := []string{}
|
|
for _, state := range []string{StateKept, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
|
if s := whole[state]; s != nil {
|
|
said = append(said, fmt.Sprintf("%s: %d manifests marking %s, %s of it marked by nothing else", state, s.Manifests, human(s.Bytes), human(s.OnlyTheirBytes)))
|
|
}
|
|
}
|
|
missing := Missing(v, recs)
|
|
if len(missing) > 0 {
|
|
said = append(said, fmt.Sprintf("%d references the records keep are not in the store", len(missing)))
|
|
}
|
|
out := map[string]any{
|
|
"store": recs.Store, "kept_builds": recs.KeptBuilds, "records": recs.Counts,
|
|
"states": whole, "repositories": repos, "missing": nonNil(missing), "said": said,
|
|
"removable": collectionRemovesThese,
|
|
}
|
|
if only != "" {
|
|
out["manifests"] = byRepo[only]
|
|
}
|
|
if recs.Note != "" {
|
|
out["note"] = recs.Note
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Collected answers store_collect: the controller's answer, and what the store's collector frees.
|
|
func Collected(v *View, a *CollectAnswer, dry bool) map[string]any {
|
|
refs := a.LetGo
|
|
if dry {
|
|
refs = a.WouldLetGo
|
|
}
|
|
gone := LetGoKeys(v, refs)
|
|
now, nowBlobs := v.Unmarked(nil)
|
|
after, afterBlobs := v.Unmarked(gone)
|
|
said := []string{}
|
|
if dry {
|
|
said = append(said, fmt.Sprintf("dry run: the controller would let go of %d of %d eligible artifacts; nothing was changed", len(a.WouldLetGo), a.Eligible))
|
|
} else {
|
|
said = append(said, fmt.Sprintf("the controller let go of %d artifacts (%d left, %d skipped)", len(a.LetGo), a.Left, a.Skipped))
|
|
if a.Stopped != "" {
|
|
said = append(said, "it stopped: "+a.Stopped)
|
|
}
|
|
}
|
|
said = append(said,
|
|
fmt.Sprintf("the nightly collector frees %s tonight as the store is now", human(now)),
|
|
fmt.Sprintf("and %s once those %d manifests are gone (%s more)", human(after), len(gone), human(after-now)))
|
|
if dry && a.Eligible > len(a.WouldLetGo) {
|
|
said = append(said, fmt.Sprintf("the controller listed %d of %d eligible: the bytes cover only those listed", len(a.WouldLetGo), a.Eligible))
|
|
}
|
|
out := map[string]any{
|
|
"dry_run": dry, "controller": a,
|
|
"collector_frees_now_bytes": now, "collector_frees_now_blobs": nowBlobs,
|
|
"collector_frees_after_bytes": after, "collector_frees_after_blobs": afterBlobs,
|
|
"manifests_gone": len(gone), "said": said,
|
|
"note": "a manifest let go of frees no bytes until the store's nightly collector runs with the store held still (ADR 0189 §4)",
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out
|
|
}
|
|
|
|
func nonNil(s []string) []string {
|
|
if s == nil {
|
|
return []string{}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func human(n int64) string {
|
|
switch {
|
|
case n >= 1<<30:
|
|
return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30))
|
|
case n >= 1<<20:
|
|
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
|
|
case n >= 1<<10:
|
|
return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
|
|
}
|
|
return fmt.Sprintf("%d B", n)
|
|
}
|
|
|
|
func text(args map[string]any, key string) string {
|
|
s, _ := args[key].(string)
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
// count is a whole-number argument, defaulted, refused below least, held at most.
|
|
func count(args map[string]any, key string, fallback, least, most int) (int, error) {
|
|
v, given := args[key]
|
|
if !given || v == nil {
|
|
return fallback, nil
|
|
}
|
|
var n int
|
|
switch x := v.(type) {
|
|
case float64:
|
|
if x != math.Trunc(x) {
|
|
return 0, fmt.Errorf("%s must be a whole number, not %v", key, x)
|
|
}
|
|
n = int(x)
|
|
case string:
|
|
i, err := strconv.Atoi(strings.TrimSpace(x))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("%s must be a whole number, not %q", key, x)
|
|
}
|
|
n = i
|
|
default:
|
|
return 0, fmt.Errorf("%s must be a whole number", key)
|
|
}
|
|
if n < least {
|
|
return 0, fmt.Errorf("%s must be at least %d", key, least)
|
|
}
|
|
return min(n, most), nil
|
|
}
|