Answer what the records keep, collect on a person's word, and name a machine's images (hq ADR 0251)

The store's tools and a machine's image pruning decide from the records, so
the controller says them: artifacts (every recorded artifact, kept and why,
eligible, collected on request), collect (the after-build sweep on demand,
a dry run unless confirmed with a why, recorded as a hand act) and images
(what a machine's declaration names, now and as last sent). The sweep is one
implementation with two sets of bounds.

internal/catalogue/verbs.go

@@ -435,6 +435,34 @@ var ControllerVerbs = []Verb{
 			"behind": "\"true\", instead of a repository: build every module the mesh holds whose source has moved " +
 				"past the commit it was built from, bases asked first",
 		}, nil, "behind")},
+	// What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51).
+	{Name: "artifacts", Description: "Every artifact the mesh recorded making that lives in its artifact store, " +
+		"each with its repository, digest, kind (image or archive) and state: kept — and why: a definition names " +
+		"it, or one of the five most recent builds of a module the mesh holds — or eligible: made, kept for no " +
+		"reason, not yet let go. Collected ones are counted, and listed only with collected. Reads only " +
+		"(novox/hq ADR 0251).",
+		Input: schema(map[string]string{
+			"repository": "only this repository's references, as <module>/<artifact> (the counts stay whole)",
+			"collected":  "\"true\": list the collected references too (they only grow; counted always)",
+		}, nil, "collected")},
+	{Name: "collect", Description: "The artifact store's sweep, now: what the mesh made and keeps for no reason " +
+		"is let go of. Without confirm a dry run — every kept archive asked about with HEADs, what would be let go " +
+		"listed, nothing held or deleted. With confirm and why: every kept archive held first, then each eligible " +
+		"artifact let go of, oldest first, and recorded collected — a hand act, recorded with its why. Bounded by " +
+		"most (500 by default, at most 5000) and 45 seconds; what is left is said. Bytes are reclaimed by the " +
+		"store's nightly collector. Never touches a digest the mesh did not record (novox/hq ADR 0189, ADR 0251).",
+		Input: schema(map[string]string{
+			"confirm": "\"true\": let go of what is eligible (needs why); without it nothing is done",
+			"why":     "with confirm: why — required, and recorded in the hand-act log",
+			"most":    "let go of at most this many (500 by default, at most 5000)",
+		}, nil, "confirm")},
+	{Name: "images", Description: "Every container image one machine's declaration names — the declaration the " +
+		"mesh would send it now and the one it was last sent — each with the resources naming it: what the " +
+		"machine keeps when it prunes its images. sent_known is false when the last one sent is not kept with " +
+		"its images. Reads only (novox/hq ADR 0251).",
+		Input: schema(map[string]string{
+			"node": "the machine's name",
+		}, []string{"node"})},
 }
 
 // schema is a JSON schema for an object of string properties, which is every argument the verbs

internal/inventory/artifacts_test.go

@@ -0,0 +1,66 @@
+package inventory
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// **Every artifact made has one state, and a kept one says why** (novox/hq ADR 0251): what the
+// store's own tools set beside what the store holds, so an operator reads why something stays.
+func TestEveryArtifactMadeHasAStateAndAKeptOneSaysWhy(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	var made []string
+	for i := 1; i <= 8; i++ {
+		made = append(made, built(t, inv, fmt.Sprintf("b%02d", i), "web", i))
+	}
+	// The definition names the oldest and the newest: the newest is kept for both reasons.
+	m := catalogue.Manifest{Module: "web", Version: "1", Resources: []map[string]any{
+		{"id": "app", "type": "container", "name": "web", "image": made[0]},
+		{"id": "next", "type": "container", "name": "web-next", "image": made[7]},
+	}}
+	if err := inv.RegisterModule(ctx, m, Source{Repository: "https://forge.invalid/web.git"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.MarkCollected(ctx, []string{made[1]}); err != nil {
+		t.Fatal(err)
+	}
+	states, err := inv.Artifacts(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(states) != len(made) {
+		t.Fatalf("%d states for %d artifacts made", len(states), len(made))
+	}
+	want := []struct {
+		state string
+		why   []string
+	}{
+		{ArtifactKept, []string{KeptByDefinition}},
+		{ArtifactCollected, nil},
+		{ArtifactEligible, nil},
+		{ArtifactKept, []string{KeptByRecentBuild}},
+		{ArtifactKept, []string{KeptByRecentBuild}},
+		{ArtifactKept, []string{KeptByRecentBuild}},
+		{ArtifactKept, []string{KeptByRecentBuild}},
+		{ArtifactKept, []string{KeptByRecentBuild, KeptByDefinition}},
+	}
+	for i, s := range states {
+		if s.Reference != made[i] {
+			t.Fatalf("state %d is of %s, want %s: the oldest first", i, s.Reference, made[i])
+		}
+		if s.State != want[i].state || fmt.Sprint(s.Why) != fmt.Sprint(want[i].why) {
+			t.Errorf("%s: %s %v, want %s %v", s.Reference, s.State, s.Why, want[i].state, want[i].why)
+		}
+	}
+	collect, err := inv.ToCollect(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(collect) != 1 || collect[0] != made[2] {
+		t.Errorf("ToCollect is %v, want only the eligible %s", collect, made[2])
+	}
+}

internal/inventory/collection.go

@@ -27,7 +27,39 @@ import (
 // release that turns out wrong can be taken.
 const KeptBuilds = 5
 
-// ToCollect is every artifact the mesh made, no longer keeps, and has not already collected.
+// Why an artifact is kept: the words `artifacts` answers (novox/hq ADR 0251).
+const (
+	// KeptByDefinition: a definition the mesh holds names it, so it could be handed to a machine now.
+	KeptByDefinition = "definition"
+	// KeptByRecentBuild: it is an artifact of one of the KeptBuilds most recent successful builds of a
+	// module the mesh still holds — somewhere a release that turns out wrong can go back to.
+	KeptByRecentBuild = "recent-build"
+	// KeptUnaddressable: it names no digest, so nothing here can speak for it, and it is kept rather
+	// than guessed about.
+	KeptUnaddressable = "unaddressable"
+)
+
+// The states an artifact the mesh recorded making is in (novox/hq ADR 0251).
+const (
+	ArtifactKept      = "kept"
+	ArtifactEligible  = "eligible"
+	ArtifactCollected = "collected"
+)
+
+// ArtifactState is one artifact the mesh recorded making, and what the records say of it now.
+type ArtifactState struct {
+	Reference string
+	State     string
+	// Why is the reasons a kept artifact is kept; empty for any other.
+	Why []string
+}
+
+// Artifacts is every artifact any successful build recorded, oldest first, each with its state:
+// kept (and why), eligible to let go, or already collected (novox/hq ADR 0189, ADR 0251).
+//
+// **One reading of the records for every question asked of them**: the sweep's ToCollect, the
+// operator's `artifacts` and the store's own tools read the same states, so what one says may go is
+// what the others say is eligible.
 //
 // Three reasons an artifact stays, and nothing else is a reason:
 //
@@ -38,57 +70,62 @@ const KeptBuilds = 5
 //     nothing beyond what a held definition names (novox/hq issue 253);
 //   - it was already collected, in which case there is nothing left to do.
 //
-// Returned in a stated order so two runs over the same records ask for the same things in the
-// same sequence, which is what makes a failed sweep safe to simply run again.
-func (i *Inventory) ToCollect(ctx context.Context) ([]string, error) {
-	keep, err := i.keptReferences(ctx)
+// In a stated order, so two runs over the same records ask for the same things in the same
+// sequence, which is what makes a failed sweep safe to simply run again.
+func (i *Inventory) Artifacts(ctx context.Context) ([]ArtifactState, error) {
+	keep, err := i.KeepSet(ctx)
 	if err != nil {
 		return nil, err
 	}
-	rows, err := i.store.Pool().Query(ctx,
-		// Every artifact of every successful build, oldest first, minus what has already been
-		// collected. A failed build published nothing, so it names nothing to remove.
-		`select b.made
-		   from build b
-		  where b.failed = '' and b.module is not null and b.module <> ''
-		  order by b.at asc, b.id asc`)
+	all, err := i.everyReferenceMade(ctx)
 	if err != nil {
 		return nil, err
 	}
-	defer rows.Close()
-
 	collected, err := i.alreadyCollected(ctx)
 	if err != nil {
 		return nil, err
 	}
-	seen := map[string]bool{}
-	var out []string
-	for rows.Next() {
-		var raw []byte
-		if err := rows.Scan(&raw); err != nil {
-			return nil, err
-		}
-		var made []Artifact
-		if err := json.Unmarshal(raw, &made); err != nil {
-			// One unreadable record must not stop the rest being collected — and an artifact this
-			// row named is simply not offered, which errs toward keeping.
-			continue
-		}
-		for _, a := range made {
-			reference := asRecorded(a.Reference)
-			if reference == "" || keep[reference] || collected[reference] || seen[reference] {
-				continue
-			}
-			seen[reference] = true
-			out = append(out, reference)
+	out := make([]ArtifactState, 0, len(all))
+	for _, reference := range all {
+		switch {
+		case len(keep[reference]) > 0:
+			out = append(out, ArtifactState{Reference: reference, State: ArtifactKept, Why: keep[reference]})
+		case collected[reference]:
+			out = append(out, ArtifactState{Reference: reference, State: ArtifactCollected})
+		default:
+			out = append(out, ArtifactState{Reference: reference, State: ArtifactEligible})
 		}
 	}
-	return out, rows.Err()
+	return out, nil
 }
 
-// keptReferences is every artifact reference the mesh still keeps, for either of the two reasons.
-func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error) {
-	keep := map[string]bool{}
+// ToCollect is every artifact the mesh made, no longer keeps, and has not already collected —
+// oldest first (Artifacts says why each other one stays).
+func (i *Inventory) ToCollect(ctx context.Context) ([]string, error) {
+	states, err := i.Artifacts(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var out []string
+	for _, s := range states {
+		if s.State == ArtifactEligible {
+			out = append(out, s.Reference)
+		}
+	}
+	return out, nil
+}
+
+// KeepSet is every artifact reference the mesh still keeps, each with the reasons it is kept.
+func (i *Inventory) KeepSet(ctx context.Context) (map[string][]string, error) {
+	keep := map[string][]string{}
+	because := func(reference, why string) {
+		for _, w := range keep[reference] {
+			if w == why {
+				return
+			}
+		}
+		keep[reference] = append(keep[reference], why)
+	}
 
 	// **Whatever a definition the mesh holds names.** Read as text rather than by walking the
 	// resource shapes: a reference may be a container's image, a bundle's source, or a field some
@@ -140,7 +177,7 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
 		}
 		for _, a := range made {
 			if reference := asRecorded(a.Reference); reference != "" {
-				keep[reference] = true
+				because(reference, KeptByRecentBuild)
 			}
 		}
 	}
@@ -148,28 +185,28 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
 		return nil, err
 	}
 
-	// And anything a manifest mentions. Done after the recent set so the scan runs over the
-	// candidates rather than over every reference ever recorded: a manifest holds a reference
-	// composed with the store's address or kept bare, so the search is for the digest within it.
+	// And anything a manifest mentions — every reference ever made, the recent ones included, so a
+	// kept artifact carries each of its reasons and the operator reads why, not only that (novox/hq
+	// ADR 0251). A manifest holds a reference composed with the store's address or kept bare, so the
+	// search is for the digest within it.
 	if len(named) > 0 {
 		all, err := i.everyReferenceMade(ctx)
 		if err != nil {
 			return nil, err
 		}
 		for _, reference := range all {
-			if keep[reference] {
-				continue
-			}
 			digest := digestIn(reference)
 			if digest == "" {
 				// Not something the store holds by digest; nothing here can speak for it, so it
 				// is kept rather than guessed about.
-				keep[reference] = true
+				if len(keep[reference]) == 0 {
+					because(reference, KeptUnaddressable)
+				}
 				continue
 			}
 			for _, text := range named {
 				if strings.Contains(text, digest) {
-					keep[reference] = true
+					because(reference, KeptByDefinition)
 					break
 				}
 			}
@@ -186,7 +223,7 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
 // reference that is kept because nothing here can speak for it is not one the store can be asked
 // about.
 func (i *Inventory) KeptArchives(ctx context.Context) ([]string, error) {
-	keep, err := i.keptReferences(ctx)
+	keep, err := i.KeepSet(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -200,10 +237,11 @@ func (i *Inventory) KeptArchives(ctx context.Context) ([]string, error) {
 	return out, nil
 }
 
-// everyReferenceMade is every artifact reference any successful build recorded.
+// everyReferenceMade is every artifact reference any successful build recorded, oldest build first.
 func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select made from build where failed = '' and module is not null and module <> ''`)
+		`select made from build where failed = '' and module is not null and module <> ''
+		  order by at asc, id asc`)
 	if err != nil {
 		return nil, err
 	}