jochen
d745a4d31e
mesh/merge-gate pass: builds artifact-store-tools → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
No record names an image the mesh did not make, so one a definition pins under an older name of the store read as unrecorded, the same word as an image nothing uses. The tool now reads every definition through the catalogue and says named-by-a-definition with the module. The store's older names are the mesh's setting older-store-names, since a catalogue names no installation.
440 lines
16 KiB
Go
440 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
)
|
|
|
|
// Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
|
|
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
|
|
func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool {
|
|
ctx := context.Background()
|
|
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
|
|
return []stdio.Tool{
|
|
{
|
|
Name: "artifact_store_repositories",
|
|
Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " +
|
|
"(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " +
|
|
"A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " +
|
|
"the store's own files; changes nothing. Replaces curl /v2/_catalog and /v2/<name>/tags/list. (r)",
|
|
Input: map[string]any{"repository": repoArg},
|
|
Run: func(args map[string]any) (any, error) {
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Repositories(v, text(args, "repository"))
|
|
},
|
|
},
|
|
{
|
|
Name: "artifact_store_usage",
|
|
Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " +
|
|
"more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " +
|
|
"frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)",
|
|
Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
top, err := count(args, "top", 10, 1, 200)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Usage(v, top), nil
|
|
},
|
|
},
|
|
{
|
|
Name: "artifact_store_references",
|
|
Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " +
|
|
"names it, or one of the five most recent builds of a module the mesh holds), named by a definition (no " +
|
|
"record names it, and a definition in the catalogue pins it by digest, under the store's current name or " +
|
|
"an older one: the module is said), the holder of a kept archive, " +
|
|
"eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " +
|
|
"a named document the controller keeps under a tag, or unrecorded — no record names it. A platform manifest " +
|
|
"takes the state of the recorded index naming it. Counted and sized by " +
|
|
"state and by repository; each manifest listed when one repository is asked. Then what the records keep that " +
|
|
"the store does not hold, and the unrecorded copies of upstream images the controller's mirrors verb may " +
|
|
"record (novox/hq ADR 0257). Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)",
|
|
Input: map[string]any{"repository": repoArg},
|
|
Run: func(args map[string]any) (any, error) {
|
|
repo := text(args, "repository")
|
|
recs, err := c.Artifacts("")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// The definitions are read while the store is: each bounded below the runtime's call limit.
|
|
read := make(chan *Definitions, 1)
|
|
go func() {
|
|
older, why := OlderNames()
|
|
defs := cat.Read(ctx, NamesOf(recs.Store, older))
|
|
defs.NamesNotRead = why
|
|
read <- defs
|
|
}()
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return References(v, recs, <-read, repo)
|
|
},
|
|
},
|
|
{
|
|
Name: "artifact_store_collect",
|
|
Description: "Collect what the mesh made and keeps for no reason (novox/hq ADR 0189, ADR 0251 §3). A dry run unless " +
|
|
"dry_run is false: it asks the controller's collect what it would let go of, and works out how many bytes the " +
|
|
"store's nightly collector would free once those are gone, beside what it frees tonight anyway. A real run needs " +
|
|
"why: it asks the controller's collect to hold every kept archive and let go of the eligible ones, recorded as a " +
|
|
"hand-act with the why; the bytes come back at the nightly collection. Never removes an unrecorded manifest, and " +
|
|
"never deletes through the store's door itself. (a)",
|
|
Input: map[string]any{
|
|
"dry_run": map[string]any{"type": "boolean", "description": "true (the default): say what would go, change nothing"},
|
|
"why": map[string]any{"type": "string", "description": "why — required for a real run, and recorded as the hand-act's reason"},
|
|
"most": map[string]any{"type": "integer", "description": "the most artifacts one real run lets go of (the controller's default when absent)"},
|
|
},
|
|
Run: func(args map[string]any) (any, error) {
|
|
dry := true
|
|
if b, given := args["dry_run"].(bool); given {
|
|
dry = b
|
|
} else if t := text(args, "dry_run"); t == "false" {
|
|
dry = false
|
|
}
|
|
why := text(args, "why")
|
|
if !dry && why == "" {
|
|
return nil, fmt.Errorf("a real collection needs why: nothing was asked of the controller")
|
|
}
|
|
most, err := count(args, "most", 0, 1, 5000)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
answer, err := c.Collect(why, !dry, most)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
v, err := s.View(ctx)
|
|
if err != nil {
|
|
return map[string]any{"controller": answer, "bytes_not_worked_out": err.Error()}, nil
|
|
}
|
|
// What a kept index names stays when an index goes (novox/hq ADR 0257): read from the
|
|
// records, as the controller's sweep reads it.
|
|
recs, err := c.Artifacts("")
|
|
if err != nil {
|
|
return map[string]any{"controller": answer, "bytes_not_worked_out": "the records could not be read: " + err.Error()}, nil
|
|
}
|
|
return Collected(v, answer, dry, Classify(v, recs, nil)), nil
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// View reads the store now.
|
|
func (s *Store) View(ctx context.Context) (*View, error) {
|
|
l, err := s.List(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
m, unread := s.Manifests(ctx, l)
|
|
return &View{L: l, M: m, Unread: unread}, nil
|
|
}
|
|
|
|
// unreadNote says what manifests that could not be read do to the numbers, when there are any.
|
|
func unreadNote(v *View) map[string]any {
|
|
if len(v.Unread) == 0 {
|
|
return nil
|
|
}
|
|
keys := make([]string, 0, len(v.Unread))
|
|
for k := range v.Unread {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
first := keys[0] + ": " + v.Unread[keys[0]]
|
|
return map[string]any{
|
|
"manifests": len(keys),
|
|
"first": first,
|
|
"means": "what these mark beyond themselves is not known, so sizes may be low and the bytes said to be " +
|
|
"freed may be high; ask again — what was read is kept, so the next call reads only the rest",
|
|
}
|
|
}
|
|
|
|
// Repositories answers artifact_store_repositories.
|
|
func Repositories(v *View, only string) (map[string]any, error) {
|
|
shared := v.Shared()
|
|
var repos []map[string]any
|
|
for _, name := range v.L.RepoNames() {
|
|
if only != "" && name != only {
|
|
continue
|
|
}
|
|
r := v.L.Repos[name]
|
|
blobs := v.RepoBlobs(name)
|
|
sharedHere := map[string]bool{}
|
|
for b := range blobs {
|
|
if shared[b] {
|
|
sharedHere[b] = true
|
|
}
|
|
}
|
|
repos = append(repos, map[string]any{
|
|
"repository": name, "tags": tagsOf(r), "manifests": len(r.Revisions),
|
|
"bytes": v.Bytes(blobs), "size": human(v.Bytes(blobs)), "shared_bytes": v.Bytes(sharedHere),
|
|
})
|
|
}
|
|
if only != "" && len(repos) == 0 {
|
|
return nil, fmt.Errorf("the store holds no repository %q", only)
|
|
}
|
|
out := map[string]any{"count": len(repos), "repositories": repos,
|
|
"note": "a repository's size is the blobs its manifests mark; a blob shared with another repository is counted in each (shared_bytes)"}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Usage answers artifact_store_usage.
|
|
func Usage(v *View, top int) map[string]any {
|
|
type sized struct {
|
|
name string
|
|
bytes int64
|
|
n int
|
|
}
|
|
var all []sized
|
|
manifests := 0
|
|
for _, name := range v.L.RepoNames() {
|
|
n := len(v.L.Repos[name].Revisions)
|
|
manifests += n
|
|
all = append(all, sized{name, v.Bytes(v.RepoBlobs(name)), n})
|
|
}
|
|
sort.SliceStable(all, func(i, j int) bool { return all[i].bytes > all[j].bytes })
|
|
var largest []map[string]any
|
|
for i, s := range all {
|
|
if i >= top {
|
|
break
|
|
}
|
|
largest = append(largest, map[string]any{"repository": s.name, "bytes": s.bytes, "size": human(s.bytes), "manifests": s.n})
|
|
}
|
|
total := v.Total()
|
|
sharedBytes := v.Bytes(v.Shared())
|
|
freed, freedBlobs := v.Unmarked(nil)
|
|
out := map[string]any{
|
|
"total_bytes": total, "total": human(total), "blobs": len(v.L.Blobs), "manifests": manifests,
|
|
"repositories": len(v.L.Repos), "largest": largest,
|
|
"shared_bytes": sharedBytes, "collector_frees_next_bytes": freed, "collector_frees_next_blobs": freedBlobs,
|
|
"said": []string{
|
|
fmt.Sprintf("the store holds %s in %d blobs, %d manifests in %d repositories", human(total), len(v.L.Blobs), manifests, len(v.L.Repos)),
|
|
fmt.Sprintf("%s is marked by more than one repository's manifests", human(sharedBytes)),
|
|
fmt.Sprintf("the nightly collector frees %s in %d blobs no manifest marks", human(freed), freedBlobs),
|
|
},
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out
|
|
}
|
|
|
|
// References answers artifact_store_references.
|
|
func References(v *View, recs *Records, defs *Definitions, only string) (map[string]any, error) {
|
|
if only != "" && v.L.Repos[only] == nil {
|
|
return nil, fmt.Errorf("the store holds no repository %q", only)
|
|
}
|
|
entries := Classify(v, recs, defs)
|
|
byRepo := map[string][]Entry{}
|
|
for _, e := range entries {
|
|
byRepo[e.Repository] = append(byRepo[e.Repository], e)
|
|
}
|
|
var repos []map[string]any
|
|
for _, name := range v.L.RepoNames() {
|
|
if only != "" && name != only {
|
|
continue
|
|
}
|
|
repos = append(repos, map[string]any{"repository": name, "states": Summarise(v, byRepo[name], entries)})
|
|
}
|
|
whole := Summarise(v, entries, entries)
|
|
said := []string{}
|
|
for _, state := range []string{StateKept, StateNamedByDefinition, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
|
if s := whole[state]; s != nil {
|
|
said = append(said, fmt.Sprintf("%s: %d manifests marking %s, %s of it marked by nothing else", state, s.Manifests, human(s.Bytes), human(s.OnlyTheirBytes)))
|
|
}
|
|
}
|
|
missing := Missing(v, recs)
|
|
if len(missing) > 0 {
|
|
said = append(said, fmt.Sprintf("%d references the records keep are not in the store", len(missing)))
|
|
}
|
|
mirrors := UnrecordedMirrors(v, entries)
|
|
if len(mirrors) > 0 {
|
|
var only int64
|
|
for _, m := range mirrors {
|
|
only += m.OnlyTheir
|
|
}
|
|
said = append(said, fmt.Sprintf("%d unrecorded copies of upstream images, %s marked by them alone: the "+
|
|
"controller's mirrors verb may record them as the mesh's (novox/hq ADR 0257), listed in unrecorded_mirrors",
|
|
len(mirrors), human(only)))
|
|
}
|
|
out := map[string]any{
|
|
"store": recs.Store, "kept_builds": recs.KeptBuilds, "records": recs.Counts,
|
|
"states": whole, "repositories": repos, "missing": nonNil(missing), "said": said,
|
|
"removable": collectionRemovesThese, "unrecorded_mirrors": mirrorsOrEmpty(mirrors),
|
|
}
|
|
if only != "" {
|
|
out["manifests"] = byRepo[only]
|
|
}
|
|
if recs.Note != "" {
|
|
out["note"] = recs.Note
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
out["definitions"] = definitionsSaid(defs)
|
|
if warn := definitionsWarning(defs); warn != "" {
|
|
// Said first: what it leaves out is a manifest a machine may run, read as unrecorded.
|
|
out["said"] = append([]string{warn}, said...)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// definitionsSaid is how the definitions were read: how many, under which names, and what was not read.
|
|
func definitionsSaid(defs *Definitions) map[string]any {
|
|
if defs == nil {
|
|
return map[string]any{"read": 0, "not_asked": "the definitions were not read"}
|
|
}
|
|
var current, older []string
|
|
for _, n := range defs.Names.Current {
|
|
current = append(current, shownName(n))
|
|
}
|
|
for _, n := range defs.Names.Older {
|
|
older = append(older, shownName(n))
|
|
}
|
|
out := map[string]any{"read": defs.Read, "store_names": nonNil(current), "older_store_names": nonNil(older)}
|
|
if defs.NotAsked != "" {
|
|
out["not_asked"] = defs.NotAsked
|
|
}
|
|
if defs.NamesNotRead != "" {
|
|
out["older_store_names_not_read"] = defs.NamesNotRead
|
|
}
|
|
if len(defs.Unread) > 0 {
|
|
out["unread"] = defs.Unread
|
|
}
|
|
return out
|
|
}
|
|
|
|
// definitionsWarning says, when any definition was not read, that a manifest it pins reads as unrecorded.
|
|
func definitionsWarning(defs *Definitions) string {
|
|
switch {
|
|
case defs == nil:
|
|
return ""
|
|
case defs.NotAsked != "":
|
|
return "the definitions could not be read (" + defs.NotAsked + "), so a manifest a definition pins " +
|
|
"and no record names reads as unrecorded here: ask again before reading unrecorded as unused"
|
|
case defs.NamesNotRead != "":
|
|
return defs.NamesNotRead + ", so a manifest a definition pins under an older name of the store and no " +
|
|
"record names reads as unrecorded here"
|
|
case len(defs.Unread) > 0:
|
|
return fmt.Sprintf("%d definitions could not be read (listed in definitions.unread), so a manifest one "+
|
|
"of them pins and no record names reads as unrecorded here: ask again — what was read is kept", len(defs.Unread))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// Collected answers artifact_store_collect: the controller's answer, and what the store's collector frees.
|
|
func Collected(v *View, a *CollectAnswer, dry bool, entries []Entry) map[string]any {
|
|
refs := a.LetGo
|
|
if dry {
|
|
refs = a.WouldLetGo
|
|
}
|
|
gone := LetGoKeys(v, refs, Spared(v, entries))
|
|
orphaned := Orphaned(v, gone)
|
|
now, nowBlobs := v.Unmarked(nil)
|
|
after, afterBlobs := v.Unmarked(gone)
|
|
said := []string{}
|
|
if dry {
|
|
said = append(said, fmt.Sprintf("dry run: the controller would let go of %d of %d eligible artifacts; nothing was changed", len(a.WouldLetGo), a.Eligible))
|
|
} else {
|
|
said = append(said, fmt.Sprintf("the controller let go of %d artifacts (%d left, %d skipped)", len(a.LetGo), a.Left, a.Skipped))
|
|
if a.Stopped != "" {
|
|
said = append(said, "it stopped: "+a.Stopped)
|
|
}
|
|
}
|
|
said = append(said,
|
|
fmt.Sprintf("the nightly collector frees %s tonight as the store is now", human(now)),
|
|
fmt.Sprintf("and %s once those %d manifests are gone (%s more)", human(after), len(gone), human(after-now)))
|
|
if len(orphaned) > 0 {
|
|
said = append(said, fmt.Sprintf("%d manifests that stay name a platform manifest this removes, and would "+
|
|
"lose it: listed in platforms_named_by_what_stays — read them before a real run", len(orphaned)))
|
|
}
|
|
if dry && a.Eligible > len(a.WouldLetGo) {
|
|
said = append(said, fmt.Sprintf("the controller listed %d of %d eligible: the bytes cover only those listed", len(a.WouldLetGo), a.Eligible))
|
|
}
|
|
out := map[string]any{
|
|
"dry_run": dry, "controller": a,
|
|
"collector_frees_now_bytes": now, "collector_frees_now_blobs": nowBlobs,
|
|
"collector_frees_after_bytes": after, "collector_frees_after_blobs": afterBlobs,
|
|
"manifests_gone": len(gone), "said": said, "platforms_named_by_what_stays": nonNil(orphaned),
|
|
"note": "a manifest let go of frees no bytes until the store's nightly collector runs with the store held still (ADR 0189 §4)",
|
|
}
|
|
if u := unreadNote(v); u != nil {
|
|
out["unread"] = u
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mirrorsOrEmpty(m []UnrecordedMirror) []UnrecordedMirror {
|
|
if m == nil {
|
|
return []UnrecordedMirror{}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func nonNil(s []string) []string {
|
|
if s == nil {
|
|
return []string{}
|
|
}
|
|
return s
|
|
}
|
|
|
|
func human(n int64) string {
|
|
switch {
|
|
case n >= 1<<30:
|
|
return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30))
|
|
case n >= 1<<20:
|
|
return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20))
|
|
case n >= 1<<10:
|
|
return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10))
|
|
}
|
|
return fmt.Sprintf("%d B", n)
|
|
}
|
|
|
|
func text(args map[string]any, key string) string {
|
|
s, _ := args[key].(string)
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
// count is a whole-number argument, defaulted, refused below least, held at most.
|
|
func count(args map[string]any, key string, fallback, least, most int) (int, error) {
|
|
v, given := args[key]
|
|
if !given || v == nil {
|
|
return fallback, nil
|
|
}
|
|
var n int
|
|
switch x := v.(type) {
|
|
case float64:
|
|
if x != math.Trunc(x) {
|
|
return 0, fmt.Errorf("%s must be a whole number, not %v", key, x)
|
|
}
|
|
n = int(x)
|
|
case string:
|
|
i, err := strconv.Atoi(strings.TrimSpace(x))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("%s must be a whole number, not %q", key, x)
|
|
}
|
|
n = i
|
|
default:
|
|
return 0, fmt.Errorf("%s must be a whole number", key)
|
|
}
|
|
if n < least {
|
|
return 0, fmt.Errorf("%s must be at least %d", key, least)
|
|
}
|
|
return min(n, most), nil
|
|
}
|