package main import ( "context" "fmt" "math" "sort" "strconv" "strings" stdio "git.novox.be/novox/mesh-sdk/go" ) // Tools are the store module's tools (novox/hq ADR 0251 §1). The first three only read; store_collect // only reads too, unless it is a real run, and then it asks the controller, which decides and deletes. func Tools(s *Store, c Controller) []stdio.Tool { ctx := context.Background() repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (/)"} return []stdio.Tool{ { Name: "store_repositories", Description: "Every repository the artifact store holds: its tags and what each names, how many manifests it holds " + "(tagged or not — the mesh pins by digest, so most are untagged) and its size, the blobs its manifests mark. " + "A blob two repositories share is counted in each, and shared_bytes says how much of a size that is. Read from " + "the store's own files and its door; changes nothing. Replaces curl /v2/_catalog and /v2//tags/list. (r)", Input: map[string]any{"repository": repoArg}, Run: func(args map[string]any) (any, error) { v, err := s.View(ctx) if err != nil { return nil, err } return Repositories(v, text(args, "repository")) }, }, { Name: "store_usage", Description: "How large the artifact store is: every blob's bytes together, the largest repositories, the bytes " + "more than one repository's manifests mark, and the bytes no manifest marks — what the store's nightly collector " + "frees next. Read from the store's own files; changes nothing. Replaces du on the store's directory. (r)", Input: map[string]any{"top": map[string]any{"type": "integer", "description": "how many of the largest repositories to name (default 10, at most 200)"}}, Run: func(args map[string]any) (any, error) { top, err := count(args, "top", 10, 1, 200) if err != nil { return nil, err } v, err := s.View(ctx) if err != nil { return nil, err } return Usage(v, top), nil }, }, { Name: "store_references", Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " + "names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " + "eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " + "a named document the controller keeps under a tag, or unrecorded — no record names it. Counted and sized by " + "state and by repository; each manifest listed when one repository is asked. Then what the records keep that " + "the store does not hold. Unrecorded manifests are never removed by any tool (novox/hq ADR 0189 §3). (r)", Input: map[string]any{"repository": repoArg}, Run: func(args map[string]any) (any, error) { repo := text(args, "repository") recs, err := c.Artifacts("") if err != nil { return nil, err } v, err := s.View(ctx) if err != nil { return nil, err } return References(v, recs, repo) }, }, { Name: "store_collect", Description: "Collect what the mesh made and keeps for no reason (novox/hq ADR 0189, ADR 0251 §3). A dry run unless " + "dry_run is false: it asks the controller's collect what it would let go of, and works out how many bytes the " + "store's nightly collector would free once those are gone, beside what it frees tonight anyway. A real run needs " + "why: it asks the controller's collect to hold every kept archive and let go of the eligible ones, recorded as a " + "hand-act with the why; the bytes come back at the nightly collection. Never removes an unrecorded manifest, and " + "never deletes through the store's door itself. (a)", Input: map[string]any{ "dry_run": map[string]any{"type": "boolean", "description": "true (the default): say what would go, change nothing"}, "why": map[string]any{"type": "string", "description": "why — required for a real run, and recorded as the hand-act's reason"}, "most": map[string]any{"type": "integer", "description": "the most artifacts one real run lets go of (the controller's default when absent)"}, }, Run: func(args map[string]any) (any, error) { dry := true if b, given := args["dry_run"].(bool); given { dry = b } else if t := text(args, "dry_run"); t == "false" { dry = false } why := text(args, "why") if !dry && why == "" { return nil, fmt.Errorf("a real collection needs why: nothing was asked of the controller") } most, err := count(args, "most", 0, 1, 5000) if err != nil { return nil, err } answer, err := c.Collect(why, !dry, most) if err != nil { return nil, err } v, err := s.View(ctx) if err != nil { return map[string]any{"controller": answer, "bytes_not_worked_out": err.Error()}, nil } return Collected(v, answer, dry), nil }, }, } } // View reads the store now. func (s *Store) View(ctx context.Context) (*View, error) { l, err := s.List(ctx) if err != nil { return nil, err } m, unread := s.Manifests(ctx, l) return &View{L: l, M: m, Unread: unread}, nil } // unreadNote says what manifests that could not be read do to the numbers, when there are any. func unreadNote(v *View) map[string]any { if len(v.Unread) == 0 { return nil } keys := make([]string, 0, len(v.Unread)) for k := range v.Unread { keys = append(keys, k) } sort.Strings(keys) first := keys[0] + ": " + v.Unread[keys[0]] return map[string]any{ "manifests": len(keys), "first": first, "means": "what these mark beyond themselves is not known, so sizes may be low and the bytes said to be " + "freed may be high; ask again — what was read is kept, so the next call reads only the rest", } } // Repositories answers store_repositories. func Repositories(v *View, only string) (map[string]any, error) { shared := v.Shared() var repos []map[string]any for _, name := range v.L.RepoNames() { if only != "" && name != only { continue } r := v.L.Repos[name] blobs := v.RepoBlobs(name) sharedHere := map[string]bool{} for b := range blobs { if shared[b] { sharedHere[b] = true } } repos = append(repos, map[string]any{ "repository": name, "tags": tagsOf(r), "manifests": len(r.Revisions), "bytes": v.Bytes(blobs), "size": human(v.Bytes(blobs)), "shared_bytes": v.Bytes(sharedHere), }) } if only != "" && len(repos) == 0 { return nil, fmt.Errorf("the store holds no repository %q", only) } out := map[string]any{"count": len(repos), "repositories": repos, "note": "a repository's size is the blobs its manifests mark; a blob shared with another repository is counted in each (shared_bytes)"} if u := unreadNote(v); u != nil { out["unread"] = u } return out, nil } // Usage answers store_usage. func Usage(v *View, top int) map[string]any { type sized struct { name string bytes int64 n int } var all []sized manifests := 0 for _, name := range v.L.RepoNames() { n := len(v.L.Repos[name].Revisions) manifests += n all = append(all, sized{name, v.Bytes(v.RepoBlobs(name)), n}) } sort.SliceStable(all, func(i, j int) bool { return all[i].bytes > all[j].bytes }) var largest []map[string]any for i, s := range all { if i >= top { break } largest = append(largest, map[string]any{"repository": s.name, "bytes": s.bytes, "size": human(s.bytes), "manifests": s.n}) } total := v.Total() sharedBytes := v.Bytes(v.Shared()) freed, freedBlobs := v.Unmarked(nil) out := map[string]any{ "total_bytes": total, "total": human(total), "blobs": len(v.L.Blobs), "manifests": manifests, "repositories": len(v.L.Repos), "largest": largest, "shared_bytes": sharedBytes, "collector_frees_next_bytes": freed, "collector_frees_next_blobs": freedBlobs, "said": []string{ fmt.Sprintf("the store holds %s in %d blobs, %d manifests in %d repositories", human(total), len(v.L.Blobs), manifests, len(v.L.Repos)), fmt.Sprintf("%s is marked by more than one repository's manifests", human(sharedBytes)), fmt.Sprintf("the nightly collector frees %s in %d blobs no manifest marks", human(freed), freedBlobs), }, } if u := unreadNote(v); u != nil { out["unread"] = u } return out } // References answers store_references. func References(v *View, recs *Records, only string) (map[string]any, error) { if only != "" && v.L.Repos[only] == nil { return nil, fmt.Errorf("the store holds no repository %q", only) } entries := Classify(v, recs) byRepo := map[string][]Entry{} for _, e := range entries { byRepo[e.Repository] = append(byRepo[e.Repository], e) } var repos []map[string]any for _, name := range v.L.RepoNames() { if only != "" && name != only { continue } repos = append(repos, map[string]any{"repository": name, "states": Summarise(v, byRepo[name], entries)}) } whole := Summarise(v, entries, entries) said := []string{} for _, state := range []string{StateKept, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} { if s := whole[state]; s != nil { said = append(said, fmt.Sprintf("%s: %d manifests marking %s, %s of it marked by nothing else", state, s.Manifests, human(s.Bytes), human(s.OnlyTheirBytes))) } } missing := Missing(v, recs) if len(missing) > 0 { said = append(said, fmt.Sprintf("%d references the records keep are not in the store", len(missing))) } out := map[string]any{ "store": recs.Store, "kept_builds": recs.KeptBuilds, "records": recs.Counts, "states": whole, "repositories": repos, "missing": nonNil(missing), "said": said, "removable": collectionRemovesThese, } if only != "" { out["manifests"] = byRepo[only] } if recs.Note != "" { out["note"] = recs.Note } if u := unreadNote(v); u != nil { out["unread"] = u } return out, nil } // Collected answers store_collect: the controller's answer, and what the store's collector frees. func Collected(v *View, a *CollectAnswer, dry bool) map[string]any { refs := a.LetGo if dry { refs = a.WouldLetGo } gone := LetGoKeys(v, refs) now, nowBlobs := v.Unmarked(nil) after, afterBlobs := v.Unmarked(gone) said := []string{} if dry { said = append(said, fmt.Sprintf("dry run: the controller would let go of %d of %d eligible artifacts; nothing was changed", len(a.WouldLetGo), a.Eligible)) } else { said = append(said, fmt.Sprintf("the controller let go of %d artifacts (%d left, %d skipped)", len(a.LetGo), a.Left, a.Skipped)) if a.Stopped != "" { said = append(said, "it stopped: "+a.Stopped) } } said = append(said, fmt.Sprintf("the nightly collector frees %s tonight as the store is now", human(now)), fmt.Sprintf("and %s once those %d manifests are gone (%s more)", human(after), len(gone), human(after-now))) if dry && a.Eligible > len(a.WouldLetGo) { said = append(said, fmt.Sprintf("the controller listed %d of %d eligible: the bytes cover only those listed", len(a.WouldLetGo), a.Eligible)) } out := map[string]any{ "dry_run": dry, "controller": a, "collector_frees_now_bytes": now, "collector_frees_now_blobs": nowBlobs, "collector_frees_after_bytes": after, "collector_frees_after_blobs": afterBlobs, "manifests_gone": len(gone), "said": said, "note": "a manifest let go of frees no bytes until the store's nightly collector runs with the store held still (ADR 0189 §4)", } if u := unreadNote(v); u != nil { out["unread"] = u } return out } func nonNil(s []string) []string { if s == nil { return []string{} } return s } func human(n int64) string { switch { case n >= 1<<30: return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30)) case n >= 1<<20: return fmt.Sprintf("%.1f MiB", float64(n)/(1<<20)) case n >= 1<<10: return fmt.Sprintf("%.1f KiB", float64(n)/(1<<10)) } return fmt.Sprintf("%d B", n) } func text(args map[string]any, key string) string { s, _ := args[key].(string) return strings.TrimSpace(s) } // count is a whole-number argument, defaulted, refused below least, held at most. func count(args map[string]any, key string, fallback, least, most int) (int, error) { v, given := args[key] if !given || v == nil { return fallback, nil } var n int switch x := v.(type) { case float64: if x != math.Trunc(x) { return 0, fmt.Errorf("%s must be a whole number, not %v", key, x) } n = int(x) case string: i, err := strconv.Atoi(strings.TrimSpace(x)) if err != nil { return 0, fmt.Errorf("%s must be a whole number, not %q", key, x) } n = i default: return 0, fmt.Errorf("%s must be a whole number", key) } if n < least { return 0, fmt.Errorf("%s must be at least %d", key, least) } return min(n, most), nil }