package main import ( "context" "encoding/json" "errors" "flag" "fmt" "os" "strings" "github.com/novox/mesh-controller/internal/artifacts" ) // What the artifact store keeps, whether each kept archive is held, and what the sweep may let go // (novox/hq issue 253, ADR 0189). // // **The question to answer before the store's collector runs for real.** The collector deletes // every blob no manifest names, and archives were published as bare blobs, so the nightly step // runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds // them as builds come; this says how far that has got — "0 unheld" is the number that lets the // dry run go. // // Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the // console through the mesh-controller seat's `command` verb (`collection --json`), which needs no // new verb in the seat's row. type collectionReport struct { // Store is the artifact store as this machine reached it; empty when it is not on the network. Store string `json:"store"` // KeptArchives is how many archives the mesh keeps, for either reason. KeptArchives int `json:"kept_archives"` // Held is how many of them the store holds by their manifest. Held int `json:"held"` // Unheld are the kept archives the collector would delete tonight if it ran for real. Unheld []string `json:"unheld"` // Missing are kept archives the store does not have at all. Missing []string `json:"missing"` // Unasked is how many could not be asked about, and why the asking stopped. Unasked int `json:"unasked"` Stopped string `json:"stopped,omitempty"` // Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet // collected — split by kind. Eligible int `json:"eligible"` EligibleImages int `json:"eligible_images"` EligibleArchives int `json:"eligible_archives"` // SafeToCollect is whether every kept archive was asked about and every one is held. SafeToCollect bool `json:"safe_to_collect"` } func collectionCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("collection", flag.ContinueOnError) asJSON := set.Bool("json", false, "answer as JSON") positionals, err := parseAround(set, args) if err != nil { return err } if len(positionals) != 0 { return errors.New("collection [--json]") } open, err := openStores(ctx) if err != nil { return err } defer open.Close() inv := open.inventory kept, err := inv.KeptArchives(ctx) if err != nil { return err } eligible, err := inv.ToCollect(ctx) if err != nil { return err } report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}} for _, reference := range eligible { if strings.Contains(reference, "/blobs/") { report.EligibleArchives++ } else { report.EligibleImages++ } } shelf, err := inv.Catalogue(ctx) if err != nil { return err } report.Store, err = artifactStoreAddress(ctx, inv, shelf, "") if err != nil { return err } if report.Store == "" { report.Unasked = len(kept) report.Stopped = "this mesh has no artifact store on its network" } else { report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report) } report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0 if *asJSON { encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") return encoder.Encode(report) } printCollection(report) return nil } // askHeld asks the store about each kept archive, stopping at the first answer that is not about // the archive: a store that cannot be reached for one cannot be for the next, and a page of // identical failures says less than one line. func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) { for i, reference := range kept { held, err := store.Held(ctx, reference) switch { case err == nil && held: report.Held++ case err == nil: report.Unheld = append(report.Unheld, reference) case errors.Is(err, artifacts.Gone): report.Missing = append(report.Missing, reference) case errors.Is(err, artifacts.ErrNotOurs): // KeptArchives names only the mesh's own; counted as unasked if one ever is not. report.Unasked++ default: return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err) } } return report.Unasked, "" } func printCollection(r collectionReport) { store := r.Store if store == "" { store = "(not on the network)" } fmt.Printf("artifact store %s\n", store) fmt.Printf("kept archives %d\n", r.KeptArchives) fmt.Printf(" held %d\n", r.Held) fmt.Printf(" unheld %d\n", len(r.Unheld)) fmt.Printf(" missing %d\n", len(r.Missing)) if r.Unasked > 0 { fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped) } fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives) if len(r.Unheld) > 0 { fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:") for _, reference := range r.Unheld { fmt.Printf(" %s\n", reference) } } if len(r.Missing) > 0 { fmt.Println("\nmissing — kept by the mesh, not in the store:") for _, reference := range r.Missing { fmt.Printf(" %s\n", reference) } } fmt.Println() if r.SafeToCollect { fmt.Println("every kept archive is held: the store's collector may run for real") } else { fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run") } }