Hold every kept archive by a manifest so the store's collector keeps it (hq issue 253)

The store's garbage-collect marks only from manifests, and archives were
published as bare blobs, so the first real collection would delete every
archive the mesh keeps. PublishArchive now puts a deterministic OCI holder
manifest (empty config, one layer) beside each archive; the sweep holds every
kept archive before it lets anything go, which backfills existing bare blobs,
and lets go of an archive holder-first. A forgotten module no longer keeps its
five recent builds (ADR 0189). `collection [--json]` reports kept archives
held/unheld and what may be let go, so the dry run can be lifted on evidence.

cmd/mesh-controller/collect.go

@@ -31,7 +31,12 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
 		fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
 		return
 	}
-	if len(references) == 0 {
+	kept, err := inv.KeptArchives(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
+		return
+	}
+	if len(references) == 0 && len(kept) == 0 {
 		return
 	}
 	shelf, err := inv.Catalogue(ctx)
@@ -59,6 +64,25 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
 	defer stop()
 	store := artifacts.Store{Address: address}
 
+	// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
+	// manifest names, and archives were published as bare blobs, so every kept archive is first
+	// held by its manifest — which backfills the ones published before holders, a few at a time
+	// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
+	// the sweep before it deletes anything: "everything kept is held" is the precondition the
+	// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
+	wrote, missing, err := holdKept(within, store, kept)
+	if wrote > 0 {
+		fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
+	}
+	if missing > 0 {
+		fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
+			"`collection` lists them\n", missing)
+	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
+		return
+	}
+
 	var done []string
 	var left, skipped int
 	for i, reference := range references {
@@ -114,6 +138,34 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
 	}
 }
 
+// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
+// Answers how many holders it wrote and how many kept archives the store does not have.
+//
+// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
+// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
+// reference the store cannot be asked about is skipped as the deletion loop skips one
+// (novox/hq issue 226).
+func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
+	for _, reference := range kept {
+		if err := ctx.Err(); err != nil {
+			return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
+		}
+		did, err := store.Hold(ctx, reference)
+		switch {
+		case err == nil:
+			if did {
+				wrote++
+			}
+		case errors.Is(err, artifacts.Gone):
+			missing++
+		case errors.Is(err, artifacts.ErrNotOurs):
+		default:
+			return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
+		}
+	}
+	return wrote, missing, nil
+}
+
 // mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
 // several times a day converges within days of this landing; small enough that no single build
 // waits on the whole backlog.

cmd/mesh-controller/collection.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+)
+
+// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
+// (novox/hq issue 253, ADR 0189).
+//
+// **The question to answer before the store's collector runs for real.** The collector deletes
+// every blob no manifest names, and archives were published as bare blobs, so the nightly step
+// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
+// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
+// dry run go.
+//
+// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
+// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
+// new verb in the seat's row.
+
+type collectionReport struct {
+	// Store is the artifact store as this machine reached it; empty when it is not on the network.
+	Store string `json:"store"`
+	// KeptArchives is how many archives the mesh keeps, for either reason.
+	KeptArchives int `json:"kept_archives"`
+	// Held is how many of them the store holds by their manifest.
+	Held int `json:"held"`
+	// Unheld are the kept archives the collector would delete tonight if it ran for real.
+	Unheld []string `json:"unheld"`
+	// Missing are kept archives the store does not have at all.
+	Missing []string `json:"missing"`
+	// Unasked is how many could not be asked about, and why the asking stopped.
+	Unasked int    `json:"unasked"`
+	Stopped string `json:"stopped,omitempty"`
+	// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
+	// collected — split by kind.
+	Eligible         int `json:"eligible"`
+	EligibleImages   int `json:"eligible_images"`
+	EligibleArchives int `json:"eligible_archives"`
+	// SafeToCollect is whether every kept archive was asked about and every one is held.
+	SafeToCollect bool `json:"safe_to_collect"`
+}
+
+func collectionCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("collection", flag.ContinueOnError)
+	asJSON := set.Bool("json", false, "answer as JSON")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	if len(positionals) != 0 {
+		return errors.New("collection [--json]")
+	}
+
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	kept, err := inv.KeptArchives(ctx)
+	if err != nil {
+		return err
+	}
+	eligible, err := inv.ToCollect(ctx)
+	if err != nil {
+		return err
+	}
+	report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
+	for _, reference := range eligible {
+		if strings.Contains(reference, "/blobs/") {
+			report.EligibleArchives++
+		} else {
+			report.EligibleImages++
+		}
+	}
+
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return err
+	}
+	report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
+	if err != nil {
+		return err
+	}
+	if report.Store == "" {
+		report.Unasked = len(kept)
+		report.Stopped = "this mesh has no artifact store on its network"
+	} else {
+		report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
+	}
+	report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
+
+	if *asJSON {
+		encoder := json.NewEncoder(os.Stdout)
+		encoder.SetIndent("", "  ")
+		return encoder.Encode(report)
+	}
+	printCollection(report)
+	return nil
+}
+
+// askHeld asks the store about each kept archive, stopping at the first answer that is not about
+// the archive: a store that cannot be reached for one cannot be for the next, and a page of
+// identical failures says less than one line.
+func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
+	for i, reference := range kept {
+		held, err := store.Held(ctx, reference)
+		switch {
+		case err == nil && held:
+			report.Held++
+		case err == nil:
+			report.Unheld = append(report.Unheld, reference)
+		case errors.Is(err, artifacts.Gone):
+			report.Missing = append(report.Missing, reference)
+		case errors.Is(err, artifacts.ErrNotOurs):
+			// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
+			report.Unasked++
+		default:
+			return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
+		}
+	}
+	return report.Unasked, ""
+}
+
+func printCollection(r collectionReport) {
+	store := r.Store
+	if store == "" {
+		store = "(not on the network)"
+	}
+	fmt.Printf("artifact store       %s\n", store)
+	fmt.Printf("kept archives        %d\n", r.KeptArchives)
+	fmt.Printf("  held               %d\n", r.Held)
+	fmt.Printf("  unheld             %d\n", len(r.Unheld))
+	fmt.Printf("  missing            %d\n", len(r.Missing))
+	if r.Unasked > 0 {
+		fmt.Printf("  not asked          %d (%s)\n", r.Unasked, r.Stopped)
+	}
+	fmt.Printf("eligible to let go   %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
+	if len(r.Unheld) > 0 {
+		fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
+		for _, reference := range r.Unheld {
+			fmt.Printf("  %s\n", reference)
+		}
+	}
+	if len(r.Missing) > 0 {
+		fmt.Println("\nmissing — kept by the mesh, not in the store:")
+		for _, reference := range r.Missing {
+			fmt.Printf("  %s\n", reference)
+		}
+	}
+	fmt.Println()
+	if r.SafeToCollect {
+		fmt.Println("every kept archive is held: the store's collector may run for real")
+	} else {
+		fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
+	}
+}

cmd/mesh-controller/main.go

@@ -73,6 +73,8 @@ func run() error {
 		return askCommand(ctx, args[1:])
 	case "builds":
 		return buildsCommand(ctx, args[1:])
+	case "collection":
+		return collectionCommand(ctx, args[1:])
 	case "plans":
 		return plansCommand(ctx, args[1:])
 	case "pin":
@@ -200,6 +202,7 @@ func usage() {
   build --behind                 build every module the mesh holds older than its source
   build --on <module>            rebuild every module that stands on this module's artifacts, bases first
   builds [<module>]              what has been built lately, and what came of it
+  collection [--json]            kept archives held/unheld by a manifest, and what the sweep may let go
   builder issue <name>           a broker account for a build machine, scoped to build work,
                                  delivered as the builder module's broker secret (module add it first)
   licence add|list|use|key       model access, under the name a person calls it

cmd/mesh-controller/seatverbs.go

@@ -198,7 +198,7 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 }
 
 // jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
-var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true}
+var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
 
 // runVerb runs this binary with the given command line and gathers what it said.
 func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {