Keep a fresh build's archives from the controller's own sweep (issue 1933)

A sweep read what it keeps before what was ever recorded, so a take-in's
build recorded between the two reads, by a concurrent take-in off the
plans' hold, was eligible and let go of seconds after the take-in read
it; a new module's build is unkept until it is registered. Now the sweep
reads the records first and keeps what a build recorded within the hour
or an open plan names, and names every reference it lets go of; a take-in
holds every artifact in the store before recording, failing on one the
store lacks; and RecordBuild clears artifact_collected as RecordMirrored
does.

cmd/mesh-controller/collect.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/novox/mesh-controller/internal/artifacts"
@@ -215,27 +217,34 @@ func collect(ctx context.Context, inv *inventory.Inventory) {
 	}
 
 	r := sweep(ctx, inv, artifacts.Store{Address: address}, references, kept, afterBuild)
+	sayCollected(os.Stderr, r)
+}
+
+// sayCollected says what one sweep after a build did.
+func sayCollected(w io.Writer, r sweepResult) {
 	if r.HoldersWritten > 0 {
-		fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
+		fmt.Fprintf(w, "the artifact store now holds %d more kept archive(s) by a manifest\n", r.HoldersWritten)
 	}
 	if r.Missing > 0 {
-		fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
+		fmt.Fprintf(w, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
 			"`collection` lists them\n", r.Missing)
 	}
 	if len(r.LetGo) > 0 {
-		fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n", len(r.LetGo))
+		// **Each one named** (novox/hq issue 1933): a count could not say which archive a machine's 404 was let go of.
+		fmt.Fprintf(w, "the artifact store let go of %d artifact(s) the mesh no longer keeps: %s\n", len(r.LetGo),
+			strings.Join(r.LetGo, ", "))
 	}
 	if r.Stopped != "" && !r.Bounded {
-		fmt.Fprintln(os.Stderr, r.Stopped)
+		fmt.Fprintln(w, r.Stopped)
 	}
 	if r.Held && r.Left > 0 {
-		fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", r.Left)
+		fmt.Fprintf(w, "%d more to collect; the next build asks again\n", r.Left)
 	}
 	if r.Skipped > 0 {
-		fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
+		fmt.Fprintf(w, "%d artifact(s) the sweep will not address were skipped\n", r.Skipped)
 	}
 	if r.Indexes > 0 {
-		fmt.Fprintf(os.Stderr, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
+		fmt.Fprintf(w, "%d eligible index(es) wait for a collect a person confirms, which takes their platforms too\n", r.Indexes)
 	}
 }
 

cmd/mesh-controller/key_takein.go

@@ -75,6 +75,32 @@ var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference
 	return artifacts.Store{Address: address}.GetBlob(ctx, reference)
 }
 
+// holdArtifact holds what a key's reference names in the artifact store: Gone when the store does not hold it, an
+// archive held by its manifest besides. A variable so a test needs no store.
+var holdArtifact = func(ctx context.Context, inv *inventory.Inventory, reference string) error {
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return err
+	}
+	address, err := artifactStoreAddress(ctx, inv, shelf, "")
+	if err != nil {
+		return err
+	}
+	if address == "" {
+		// A mesh being raised has no store on its network yet (as the sweep, which collects nothing there): nothing
+		// can be held, and waiting for the store would keep the store itself from being taken in. Said, never silent.
+		fmt.Printf("the artifact store is on no network the controller reaches, so %s is taken in without being held\n",
+			reference)
+		return nil
+	}
+	store := artifacts.Store{Address: address}
+	if err := store.Holds(ctx, reference); err != nil {
+		return err
+	}
+	_, err = store.Hold(ctx, reference)
+	return err
+}
+
 // predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head.
 type predefinedHead struct {
 	List       catalogue.PredefinedList
@@ -223,7 +249,8 @@ func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t k
 		return catalogue.Manifest{}, inventory.Build{}, err
 	}
 	kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous,
-		archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }})
+		archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) },
+		hold:    func(ctx context.Context, reference string) error { return holdArtifact(ctx, inv, reference) }})
 	if err != nil {
 		return catalogue.Manifest{}, kept, err
 	}
@@ -251,6 +278,8 @@ type keyContext struct {
 	list     catalogue.PredefinedList
 	previous []inventory.Build
 	archive  func(ctx context.Context, reference string) ([]byte, error)
+	// hold holds one artifact the key names in the artifact store, artifacts.Gone when the store does not hold it.
+	hold func(ctx context.Context, reference string) error
 }
 
 // keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root.
@@ -318,6 +347,27 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
 			return inventory.Build{}, catalogue.Manifest{}, err
 		}
 	}
+	// **Every artifact held in the store before anything is recorded** (novox/hq issue 1933): a definition naming an
+	// archive the store does not hold is sent, and every machine assigned the module fails its apply on the 404. An
+	// archive is held by its manifest, so the store's own collector keeps it too.
+	if kc.hold != nil {
+		for _, a := range k.Build.Artifacts {
+			reference := catalogue.Recorded(a.Reference)
+			if _, ours := catalogue.InArtifactStore(reference); !ours {
+				continue
+			}
+			err := kc.hold(ctx, reference)
+			switch {
+			case err == nil, errors.Is(err, artifacts.ErrNotOurs):
+			case errors.Is(err, artifacts.Gone):
+				return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s for its "+
+					"artifact %q, which is not in the artifact store: nothing was recorded or registered, and the key "+
+					"must be built again", t.Module, reference, a.Name)
+			default:
+				return waits("the artifact store could not hold %s's artifact %q (%s): %v", t.Module, a.Name, reference, err)
+			}
+		}
+	}
 	built, err := builtOfKey(ctx, kc.archive, manifest, k)
 	if err != nil {
 		return waits("%v", err)

cmd/mesh-controller/sweep_keeps_fresh_test.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/novox/mesh-controller/internal/artifacts"
+)
+
+// 1933/AC2: a take-in holds every artifact of the key in the artifact store before anything is recorded, and an archive
+// the store does not hold fails it, "not in the artifact store", with no build to record: on 2026-10-24 00:37 the
+// indexer's program blob was missing, the take-in registered it anyway and novox failed every apply on its 404.
+func TestATakeInHoldsEveryArtifactAndFailsOnOneTheStoreLacks(t *testing.T) {
+	program := "artifact-store://x/tools/blobs/" + shaOf("t")
+	image := "artifact-store://x/img@" + shaOf("i")
+	keys, trunk := aKeyOf(tsBundle, 3,
+		sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), Reference: program},
+		sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"), Reference: image})
+	take := keyTake{Module: "x", Trunk: trunk}
+
+	var held []string
+	holds := func(_ context.Context, reference string) error { held = append(held, reference); return nil }
+	if _, _, err := buildOfKey(t.Context(), keys, take, keyContext{hold: holds}); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(held, " ") != program+" "+image {
+		t.Errorf("the take-in held %v; want both of the key's artifacts", held)
+	}
+
+	lacks := func(_ context.Context, reference string) error {
+		if reference == program {
+			return artifacts.Gone
+		}
+		return nil
+	}
+	kept, _, err := buildOfKey(t.Context(), keys, take, keyContext{hold: lacks})
+	if err == nil || errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "not in the artifact store") ||
+		!strings.Contains(err.Error(), program) {
+		t.Errorf("a take-in whose program archive the store lacks: %v; want a failure naming it, not in the artifact store", err)
+	}
+	if kept.ID != "" || len(kept.Made) > 0 {
+		t.Errorf("a take-in that failed made a build to record: %s %v", kept.ID, kept.Made)
+	}
+
+	unreachable := func(context.Context, string) error { return errors.New("connection refused") }
+	if _, _, err := buildOfKey(t.Context(), keys, take, keyContext{hold: unreachable}); !errors.Is(err, errKeyWaits) {
+		t.Errorf("a store that cannot be asked: %v; want a wait", err)
+	}
+}
+
+// 1933/AC1: the sweep after a build names every reference it let go of, not only how many, so a let-go that broke a
+// machine is read from the controller's log.
+func TestTheSweepSaysEveryReferenceItLetGoOf(t *testing.T) {
+	var out bytes.Buffer
+	sayCollected(&out, sweepResult{Held: true, LetGo: []string{
+		"artifact-store://route-proxy/files/blobs/sha256:58bd41dc", "artifact-store://old/app@sha256:01"}})
+	for _, reference := range []string{"artifact-store://route-proxy/files/blobs/sha256:58bd41dc",
+		"artifact-store://old/app@sha256:01"} {
+		if !strings.Contains(out.String(), reference) {
+			t.Errorf("the sweep's words do not name %s:\n%s", reference, out.String())
+		}
+	}
+}