Merge pull request 'Answer held and say artifacts-gone; write the grants again; integration tests match R4g (issues 1937, 1938, 1931)' (#453) from fix/1937-done-key-blobs into main

Reviewed-on: #453

cmd/mesh-controller/asker_retry_integration_test.go

@@ -23,7 +23,7 @@ import (
 func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
 	open := aMesh(t)
 	ctx := t.Context()
-	asked := asksRecorded(t)
+	asksRecorded(t)
 	planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
 	twoTiers(t, open)
 	before := time.Now().UTC().Add(-time.Hour)
@@ -123,12 +123,9 @@ func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *test
 		t.Fatal(err)
 	}
 	m := p.Modules["a"]
-	if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
+	if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" {
 		t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
 	}
-	if len(*asked) != 1 {
-		t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
-	}
 	if got := store[w.Ask]; got.Acted != "done" {
 		t.Fatalf("the ask ended %q", got.Acted)
 	}

cmd/mesh-controller/away_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -15,6 +16,7 @@ import (
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/builderkeys"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 	"github.com/novox/mesh-controller/internal/testbus"
@@ -68,6 +70,16 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 	var mu sync.Mutex
 	var sends [][]string
 	everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it
+	// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look
+	// goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask
+	// was where the walk paused).
+	wholeKeys := openPlanKeys
+	var holdB atomic.Bool
+	holdB.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{all, "b", &holdB, ""}, done, err
+	}
 	digests := 0
 	wasSend := sendRollout
 	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
@@ -189,7 +201,9 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 	if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
 		t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
 	}
-	// Tier 1's module is taken from its key, which the walk's look does by itself.
+	// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
+	holdB.Store(false)
+	askToLook()
 	eventually(t, "tier 1 sent to anchor alone", func() bool {
 		mu.Lock()
 		defer mu.Unlock()
@@ -211,3 +225,39 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 		t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
 	}
 }
+
+// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of
+// the fingerprint fp when that is given.
+type keysHolding struct {
+	planKeyReads
+	module string
+	held   *atomic.Bool
+	fp     string
+}
+
+func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
+	if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) {
+		return builderkeys.Key{Module: module, Fingerprint: fp}, nil
+	}
+	return k.planKeyReads.Build(ctx, module, fp)
+}
+
+// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared,
+// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of
+// the test, and the flag is set until the test lowers it.
+func holdKeys(t *testing.T, modules ...string) *atomic.Bool {
+	t.Helper()
+	held := &atomic.Bool{}
+	held.Store(true)
+	was := openPlanKeys
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := was()
+		var keys planKeyReads = all
+		for _, m := range modules {
+			keys = keysHolding{keys, m, held, ""}
+		}
+		return keys, done, err
+	}
+	t.Cleanup(func() { openPlanKeys = was })
+	return held
+}

cmd/mesh-controller/backups_before_send_integration_test.go

@@ -211,8 +211,12 @@ func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItA
 	}
 	seats["laptop"].answers("notes", true, "")
 	advancePlans(ctx, open)
-	if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 {
+	// One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key
-		t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked)
+	// build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes
+	// it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key.
+	if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 ||
+		got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" {
+		t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked)
 	}
 	if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
 		t.Fatalf("the condition stays: %+v", said)

cmd/mesh-controller/batches_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"slices"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -65,7 +66,7 @@ func cutAt(t *testing.T, open *stores, now time.Time) {
 func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
 	open := windowed(t)
 	everyHeadWaits(t)
-	asked := asksWithPaths(t)
+	asksWithPaths(t)
 	claude := catalogueMerge("553b7191claude", "app", t0)
 	dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
 	hear(t, open, claude, t0.Add(time.Second))
@@ -96,8 +97,12 @@ func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
 	if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
 		t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
 	}
-	if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
+	// The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at
-		t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
+	// which commit: the one the branch holds, which is the later merge the walk carries.
+	for _, m := range w.Tiers[w.Tier] {
+		if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit {
+			t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s)
+		}
 	}
 	// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
 	hear(t, open, claude, t0.Add(5*time.Minute))
@@ -131,11 +136,10 @@ func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
 			t.Fatalf("the walk does not build %s: %v", r, w.Modules)
 		}
 	}
-	// Each module's ask is recorded at its own repository's commit.
+	// Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build).
-	for _, r := range []string{"one", "two", "three"} {
+	for _, r := range w.Tiers[w.Tier] {
-		q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
+		if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r {
-		if err != nil || !found || q.Commit != "c-"+r {
+			t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s)
-			t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
 		}
 	}
 }
@@ -340,7 +344,7 @@ func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
 func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
 	open := windowed(t)
 	everyHeadWaits(t)
-	asked := asksWithPaths(t)
+	asksWithPaths(t)
 	ctx := t.Context()
 	// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
 	oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
@@ -367,9 +371,10 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
 		!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
 		t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
 	}
-	if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
+	// The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the
-		middle.Commit} {
+	// module is taken from its key at the commit the merge was walked on.
-		t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
+	if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit {
+		t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules)
 	}
 	// Retried, the failed walk would name merges the search answers now.
 	if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
@@ -508,6 +513,14 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
 func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
 	open := windowed(t)
 	asked := asksWithPaths(t)
+	// The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with
+	// the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other.
+	wholeKeys, held := openPlanKeys, &atomic.Bool{}
+	held.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err
+	}
 	ctx := t.Context()
 	hear(t, open, repoMerge("one", "c1", t0), t0)
 	cutAt(t, open, t0.Add(2*time.Minute))
@@ -541,7 +554,11 @@ func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
 			t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
 		}
 	}
-	done := started[0]
+	// Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale.
+	done, err := open.inventory.PlanByID(ctx, started[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
 	done.State = inventory.PlanDone
 	if err := open.inventory.SavePlan(ctx, &done); err != nil {
 		t.Fatal(err)
@@ -624,6 +641,14 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
 	// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
 	// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
 	catalogueWalk := ws[0]
+	// The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the
+	// stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk.
+	wholeKeys, held := openPlanKeys, &atomic.Bool{}
+	held.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err
+	}
 	hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
 	cutAt(t, open, t0.Add(6*time.Minute))
 	ws, _ = walks(t, open)
@@ -642,7 +667,11 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
 	if len(*asked) != before {
 		t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
 	}
-	own2 := ws[0]
+	// Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale.
+	own2, err := open.inventory.PlanByID(ctx, ws[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
 	own2.State = inventory.PlanDone
 	if err := open.inventory.SavePlan(ctx, &own2); err != nil {
 		t.Fatal(err)

cmd/mesh-controller/beside_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -15,6 +16,14 @@ import (
 func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
 	open := windowed(t)
 	asksWithPaths(t)
+	// Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in
+	// builder's key done at once, one step would carry a one-module walk to done.
+	wholeKeys, held := openPlanKeys, &atomic.Bool{}
+	held.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err
+	}
 	ctx := t.Context()
 	hear(t, open, repoMerge("one", "c1", t0), t0)
 	cutAt(t, open, t0.Add(90*time.Second))

cmd/mesh-controller/busobjects_test.go

@@ -46,25 +46,33 @@ func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
 	return broker.Consumer{}
 }
 
-// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
+// aLateHolder is a module holding a build seat (node-build-go) on anchor, assigned after the controller started, and
-// started, and the worker its seat's queue should have for it.
+// the worker its seat's queue should have for it: a build seat is pulled by kind, so the worker is the one for the
+// seat's first accepted kind (the build agent's seat, which had one worker for all, is gone with the builder).
 func aLateHolder(t *testing.T, open *stores) broker.Consumer {
 	t.Helper()
+	// With an account to read, as a holder has: a module with none is no bus user, so its seat's workers are not derived.
 	register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
-		Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
+		OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/late-builder/broker"}},
+		Claims:     []catalogue.Claim{{Name: "node-build-go", Scope: catalogue.ScopeNode}}})
 	if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
 		t.Fatal(err)
 	}
+	_, workers, err := seatTrafficObjects(t.Context(), open.inventory)
+	if err != nil {
+		t.Fatal(err)
+	}
 	for _, s := range inventory.MeshSeats() {
-		if s.Name == "node-build-agent" {
+		if s.Name != "node-build-go" || len(s.Accepts) == 0 {
-			c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
+			continue
-			if !needed {
+		}
-				t.Fatal("the build agent's seat needs no worker")
+		for _, w := range workers {
+			if w.Name == broker.WorkerName(s.Name, s.Accepts[0]) {
+				return w
 			}
-			return c
 		}
 	}
-	t.Fatal("the mesh declares no build agent's seat")
+	t.Fatal("the build seat's holder is derived no worker")
 	return broker.Consumer{}
 }
 

cmd/mesh-controller/delivery_integration_test.go

@@ -72,7 +72,10 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 	}
 
 	// No holder on record: the merge starts its walk, as before.
+	// A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no
+	// build), so each walk that starts is stepped once before its takes are counted.
 	p := merge("c1aaaaaaaa", "modules/app/index.ts")
+	advanceHeld(ctx, open)
 	if p.Waiting() || len(*asked) != 1 {
 		t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
 	}
@@ -116,6 +119,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 	// ADR 0276): the merge's batch is cut once it ended.
 	finish(p.ID)
 	p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
+	advanceHeld(ctx, open)
 	if p.Waiting() || len(*asked) != 3 {
 		t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
 	}
@@ -157,7 +161,9 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 		t.Fatal(err)
 	}
 	asks := len(*asked)
-	if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
+	p = merge("c6ffffffff", "modules/app/index.ts")
+	advanceHeld(ctx, open)
+	if p.Waiting() || len(*asked) != asks+1 {
 		t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
 	}
 }

cmd/mesh-controller/held.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The verb `held` and the event `artifacts-gone` (novox/hq issue 1937).
+//
+// **A done key can name a blob the store no longer holds.** A build module pushes a bundle or an archive as a bare blob
+// (ADR 0498 §1); the store's nightly collector deletes every blob no manifest names; and only a take-in or a sweep holds
+// one by its manifest. A key the controller has not taken in by the next collection names a blob that is gone. On
+// 2026-10-24 that was node-tools', mesh-host's and the forge bridge's. Since issue 1933 the take-in refuses such a key;
+// these two let the builder build it again: `held` before it reuses a done key, `artifacts-gone` when a take-in found it.
+
+// heldMost is the most references one ask of held judges: a builder asks for one key's artifacts at a time.
+const heldMost = 64
+
+// heldAsk asks the store whether it holds one reference: nil, artifacts.Gone or another error. A variable so a test
+// needs no store.
+var heldAsk = func(ctx context.Context, address, reference string) error {
+	return artifacts.Store{Address: address}.Holds(ctx, reference)
+}
+
+// heldAddress is the artifact store's address the serving controller asks. A variable so a test needs no records.
+var heldAddress = func(ctx context.Context) (string, error) {
+	d := doctorFrom
+	if d == nil || d.open == nil || d.open.inventory == nil {
+		return "", errors.New("this controller is not serving, so it asks the artifact store nothing: ask again, and " +
+			"the serving controller answers")
+	}
+	shelf, err := d.open.inventory.Catalogue(ctx)
+	if err != nil {
+		return "", err
+	}
+	address, err := artifactStoreAddress(ctx, d.open.inventory, shelf, "")
+	if err != nil {
+		return "", err
+	}
+	if address == "" {
+		return "", errors.New("the artifact store is on no network the controller reaches")
+	}
+	return address, nil
+}
+
+// heldAnswer is the verb `held`: each reference given, held, gone (the store answered 404) or unknown with why. It
+// holds and writes nothing: a reference the store holds and no record names is left for the take-in to hold.
+func heldAnswer(ctx context.Context, given string) (any, error) {
+	var refs []string
+	for _, r := range strings.Split(given, ",") {
+		if r = strings.TrimSpace(r); r != "" && !slices.Contains(refs, r) {
+			refs = append(refs, r)
+		}
+	}
+	if len(refs) == 0 {
+		return nil, errors.New("held judges the references given, and none was")
+	}
+	if len(refs) > heldMost {
+		return nil, fmt.Errorf("held judges at most %d references at once, and %d were given", heldMost, len(refs))
+	}
+	for _, r := range refs {
+		if _, ours := catalogue.InArtifactStore(r); !ours {
+			return nil, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r)
+		}
+	}
+	address, err := heldAddress(ctx)
+	if err != nil {
+		return nil, err
+	}
+	asking, stop := context.WithTimeout(ctx, 20*time.Second)
+	defer stop()
+	a := sdkdelivery.HeldAnswer{Store: address, Held: []string{}, Gone: []string{}}
+	for _, r := range refs {
+		switch err := heldAsk(asking, address, r); {
+		case err == nil:
+			a.Held = append(a.Held, r)
+		case errors.Is(err, artifacts.Gone):
+			a.Gone = append(a.Gone, r)
+		default:
+			if a.Unknown == nil {
+				a.Unknown = map[string]string{}
+			}
+			a.Unknown[r] = err.Error()
+		}
+	}
+	out := fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked", address, len(a.Held), len(refs))
+	if len(a.Gone) > 0 {
+		out += "; gone: " + strings.Join(a.Gone, ", ")
+	}
+	if len(a.Unknown) > 0 {
+		out += fmt.Sprintf("; %d could not be asked", len(a.Unknown))
+	}
+	return verbAnswer{Output: out + "\n", OK: true, Answer: a}, nil
+}
+
+// sayArtifactsGone says `artifacts-gone` for a builder's key the take-in refused because the store does not hold what
+// it names (issue 1937): the builder hears it and builds the key again. Said from the serving controller only, and never
+// in the take-in's way: a bus that cannot carry it is said in the journal.
+var sayArtifactsGone = func(ctx context.Context, k builderkeys.Key, module string, gone []string) {
+	bus := checkEvents
+	if bus == nil || len(gone) == 0 {
+		return
+	}
+	key, _ := sdkdelivery.BuilderBuildKey(module, k.Fingerprint)
+	raw, err := sdkdelivery.EncodeArtifactsGone(sdkdelivery.ArtifactsGone{Module: module, Key: key,
+		Fingerprint: k.Fingerprint, Gone: gone, At: time.Now().UTC()})
+	if err != nil {
+		fmt.Printf("%s: its key names archives the store does not hold, and that could not be said: %v\n", module, err)
+		return
+	}
+	saying, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+	defer stop()
+	if err := bus.PublishSeatEvent(saying, link.MeshControllerSeat, link.KeyArtifactsGone, raw); err != nil {
+		fmt.Printf("%s: its key names archives the store does not hold, and %s could not be said: %v\n",
+			module, link.KeyArtifactsGone, err)
+	}
+}

cmd/mesh-controller/held_test.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// 1937/AC1: held answers each reference held, gone (the store's 404) or unknown, asks only HEADs, and its answer is what
+// the builder reads (mesh-sdk delivery.DecodeHeldAnswer).
+func TestHeldAnswersEachReferenceHeldOrGone(t *testing.T) {
+	present := "artifact-store://node-tools/runtime/blobs/" + shaOf("held")
+	missing := "artifact-store://node-tools/runtime/blobs/" + shaOf("gone")
+	broken := "artifact-store://mesh-host/host-arch/blobs/" + shaOf("broken")
+	address, ask := heldAddress, heldAsk
+	t.Cleanup(func() { heldAddress, heldAsk = address, ask })
+	heldAddress = func(context.Context) (string, error) { return "store.test:5100", nil }
+	var asked []string
+	var mu sync.Mutex
+	heldAsk = func(_ context.Context, at, reference string) error {
+		mu.Lock()
+		asked = append(asked, at+" "+reference)
+		mu.Unlock()
+		switch reference {
+		case missing:
+			return artifacts.Gone
+		case broken:
+			return errors.New("the artifact store answered 500")
+		}
+		return nil
+	}
+	got, err := heldAnswer(t.Context(), present+", "+missing+","+broken+","+present)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := json.Marshal(got)
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, err := sdkdelivery.DecodeHeldAnswer(raw, []string{present, missing, broken})
+	if err != nil {
+		t.Fatalf("the builder cannot read held's answer: %v\n%s", err, raw)
+	}
+	if !slices.Equal(a.Held, []string{present}) || !slices.Equal(a.Gone, []string{missing}) || a.Unknown[broken] == "" {
+		t.Errorf("held answered %+v", a)
+	}
+	if len(asked) != 3 {
+		t.Errorf("held asked the store %d times for three distinct references: %v", len(asked), asked)
+	}
+	if !strings.Contains(got.(verbAnswer).Output, missing) {
+		t.Errorf("held's words do not name the gone reference: %s", got.(verbAnswer).Output)
+	}
+
+	for _, refused := range []string{"", "registry.example/x@sha256:00"} {
+		if _, err := heldAnswer(t.Context(), refused); err == nil {
+			t.Errorf("held took %q", refused)
+		}
+	}
+	// A verb answered in the serving process, never by a command line.
+	if !inProcess["held"] {
+		t.Error("held is not answered by the serving controller")
+	}
+	if _, err := argvFor("held", map[string]any{"references": present}); err == nil {
+		t.Error("held composed a command line")
+	}
+}
+
+// saidGone keeps the artifacts-gone events said.
+type saidGone struct {
+	link.Bus // only PublishSeatEvent is said on
+	mu       sync.Mutex
+	said     []sdkdelivery.ArtifactsGone
+}
+
+func (s *saidGone) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
+	if seat != link.MeshControllerSeat || event != link.KeyArtifactsGone {
+		return errors.New("said under the wrong seat or name: " + seat + " " + event)
+	}
+	e, err := sdkdelivery.DecodeArtifactsGone(body)
+	if err != nil {
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.said = append(s.said, e)
+	return nil
+}
+
+// 1937/AC2: a take-in refused for a gone archive names every gone one and says artifacts-gone with the module, the key,
+// its fingerprint and the gone references, so the builder builds the key again; a hand registration says nothing.
+func TestARefusedTakeInSaysArtifactsGone(t *testing.T) {
+	program := "artifact-store://x/tools/blobs/" + shaOf("t")
+	image := "artifact-store://x/img@" + shaOf("i")
+	keys, trunk := aKeyOf(tsBundle, 3,
+		sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), Reference: program},
+		sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"), Reference: image})
+	bus := &saidGone{}
+	was := checkEvents
+	checkEvents = bus
+	t.Cleanup(func() { checkEvents = was })
+	gone := func(context.Context, string) error { return artifacts.Gone }
+
+	_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{hold: gone})
+	if err == nil || !strings.Contains(err.Error(), program) || !strings.Contains(err.Error(), image) ||
+		!strings.Contains(err.Error(), "not in the artifact store") {
+		t.Fatalf("a take-in of a key whose archives are gone: %v", err)
+	}
+	if len(bus.said) != 1 {
+		t.Fatalf("said %d artifacts-gone events; want one", len(bus.said))
+	}
+	e := bus.said[0]
+	wantKey, _ := sdkdelivery.BuilderBuildKey("x", keyFP)
+	if e.Module != "x" || e.Fingerprint != keyFP || e.Key != wantKey || !slices.Equal(e.Gone, []string{image, program}) {
+		t.Errorf("said %+v", e)
+	}
+
+	bus.said = nil
+	_, _, _ = buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk, Hand: map[string]string{"tools": program}},
+		keyContext{hold: gone})
+	if len(bus.said) != 0 {
+		t.Errorf("a registration by hand said %v: no builder's key was refused", bus.said)
+	}
+}

cmd/mesh-controller/key_takein.go

@@ -351,6 +351,9 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
 	// archive the store does not hold is sent, and every machine assigned the module fails its apply on the 404. An
 	// archive is held by its manifest, so the store's own collector keeps it too.
 	if kc.hold != nil {
+		// **Every gone archive named, and said** (issue 1937): the builder hears `artifacts-gone` and builds the key
+		// again, so the refusal is the last word on this key, not a plan that fails at every retry.
+		var gone, goneNames []string
 		for _, a := range k.Build.Artifacts {
 			reference := catalogue.Recorded(a.Reference)
 			if _, ours := catalogue.InArtifactStore(reference); !ours {
@@ -360,13 +363,21 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
 			switch {
 			case err == nil, errors.Is(err, artifacts.ErrNotOurs):
 			case errors.Is(err, artifacts.Gone):
-				return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s for its "+
+				gone = append(gone, reference)
-					"artifact %q, which is not in the artifact store: nothing was recorded or registered, and the key "+
+				goneNames = append(goneNames, fmt.Sprintf("%s for its artifact %q", reference, a.Name))
-					"must be built again", t.Module, reference, a.Name)
 			default:
 				return waits("the artifact store could not hold %s's artifact %q (%s): %v", t.Module, a.Name, reference, err)
 			}
 		}
+		if len(gone) > 0 {
+			if t.Hand == nil {
+				sayArtifactsGone(ctx, k, t.Module, gone)
+			}
+			return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s, not in "+
+				"the artifact store: nothing was recorded or registered, and the key must be built "+
+				"again (said as %s, which the builder builds it again on)", t.Module, strings.Join(goneNames, " and "),
+				link.KeyArtifactsGone)
+		}
 	}
 	built, err := builtOfKey(ctx, kc.archive, manifest, k)
 	if err != nil {

cmd/mesh-controller/one_group_one_plan_test.go

@@ -94,8 +94,10 @@ func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *t
 			if plan.Delivery != nil && (plan.Delivery.Awaits != "" || plan.Waiting()) {
 				t.Fatalf("a plan carrying the controller waits for %q's word", plan.Delivery.Awaits)
 			}
-			if plan.Modules["mesh-controller"].Build == "" || len(*asked) != 1 {
+			// The cut takes the first tier from the controller's key at its commit and asks no build; the take-in itself is a
-				t.Fatalf("the first tier is asked once: build %q, %d ask(s) %v", plan.Modules["mesh-controller"].Build, len(*asked), *asked)
+			// later look's, which this test does not run, so nothing is recorded as asked.
+			if c := plan.Modules["mesh-controller"]; !c.Key || c.State != "asked" || c.KeyAt != "k1" || len(*asked) != 0 {
+				t.Fatalf("the first tier is not taken from its key once: %+v, %d ask(s) %v", c, len(*asked), *asked)
 			}
 			// Every merge is answered by that one plan.
 			for _, repository := range []string{"novox/mesh-controller", "novox/mesh-catalog"} {

cmd/mesh-controller/pending_integration_test.go

@@ -151,7 +151,9 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
 	if len(pending) != 1 || pending[0].State != inventory.PendingExpired {
 		t.Fatalf("pending after a failed build: %+v", pending)
 	}
-	for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed",
+	// Its words since R4g (whyNotBuilt): the build, when it was heard, and that it failed (issue 1938: kept from an older
+	// run until the inventory changed beneath it).
+	for _, want := range []string{"expired", "modules/sensors in novox/mesh-catalog; its last build, b-modules/sensors at", ", failed: go build",
 		"undefined: sensorsRead", "nothing was assigned", "build \"true\""} {
 		if !strings.Contains(pending[0].Note, want) {
 			t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note)
@@ -266,9 +268,18 @@ func TestAPlansBuildRequestIsThePlans(t *testing.T) {
 	if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
 		t.Fatal(err)
 	}
+	// A walk asks no build, so it leaves no build request (askModule takes the module from its key): the plan's app is
+	// taken from the key at the commit the plan carries.
 	requests, err := open.inventory.RequestsNamed(ctx, "app")
-	if err != nil || len(requests) != 1 || requests[0].For != "plan" {
+	if err != nil || len(requests) != 0 {
-		t.Fatalf("the plan's request: %+v %v", requests, err)
+		t.Fatalf("a request was recorded for the plan: %+v %v", requests, err)
+	}
+	recent, err := open.inventory.RecentPlans(ctx, 5)
+	if err != nil || len(recent) != 1 {
+		t.Fatalf("the plans: %+v %v", recent, err)
+	}
+	if app := recent[0].Modules["app"]; app == nil || !app.Key || app.KeyAt != "c1aaaaaaaa" {
+		t.Fatalf("the plan's app is not taken from its key at the commit it carries: %+v", app)
 	}
 }
 

cmd/mesh-controller/plan_clock_integration_test.go

@@ -15,6 +15,9 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
 	ctx := t.Context()
 	inv := open.inventory
 	asked := asksRecorded(t)
+	// app's key is not built, so the plan stands on its tier as it did while the build ran; with the stand-in builder's
+	// key done at once, one step would carry it to done.
+	holdKeys(t, "app")
 	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
 		inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
 			BuiltFrom: "c0", Head: "c0"}); err != nil {
@@ -27,11 +30,12 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
 	}
 	advanceHeld(ctx, open)
 	recent, err := inv.RecentPlans(ctx, 1)
-	if err != nil || len(recent) != 1 || len(*asked) != 1 {
+	if err != nil || len(recent) != 1 || len(*asked) != 0 {
 		t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked)
 	}
 	before := recent[0]
-	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
+	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" ||
+		!before.Modules["app"].Key || before.Modules["app"].KeyAt != "c1aaaaaaaa" {
 		t.Fatalf("the plan is not waiting on its build: %+v", before)
 	}
 
@@ -46,7 +50,7 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
 		t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
 			before.Revision, after.Revision, before.Updated, after.Updated)
 	}
-	if len(*asked) != 1 {
+	if len(*asked) != 0 {
-		t.Fatalf("advancing asked again: %v", *asked)
+		t.Fatalf("advancing took the key in: %v", *asked)
 	}
 }

cmd/mesh-controller/reconnects_integration_test.go

@@ -52,7 +52,9 @@ func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
 	if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
 		t.Fatalf("hand-acts answered %+v", a)
 	}
-	_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
+	// A read-only verb that runs in the serving process, as a verb that reads the bus does: it uses the
+	// serving connection and opens none. (A verb run as a command re-executes this test binary, which never ends.)
+	_, _ = handlers["dead-letters"](context.Background(), json.RawMessage(`{}`))
 	after, _ := bus.Varz(nil)
 	if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
 		t.Fatalf("the serving controller opened %d connection(s) of its own", opened)

cmd/mesh-controller/replay614_test.go

@@ -63,11 +63,14 @@ func TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles(t *testing.T) {
 	m := orderMember{ID: "novox/mesh-sdk@new", Repository: "novox/mesh-sdk", Head: "new",
 		Paths: []string{"src/provisioner/index.ts", "package.json"}}
 	s := pullScope(m.pull(), entries, read, edges)
-	want := [][]string{{"mesh-sdk"}, {"mesh-tools"}, {"minio", "umami"}}
+	// Since R4g no bundle is compiled in a toolchain image (526eed99, issue 1851 C1b: the toolchain edge is gone), so a
+	// new SDK plans itself and what stands on its package; the TypeScript bundles move by their own build keys, whose
+	// fingerprints name what they are built from (issue 1938: this test was kept from older runs until then).
+	want := [][]string{{"mesh-sdk"}, {"mesh-tools"}}
 	if !reflect.DeepEqual(s.Reach.Plan.Tiers, want) {
 		t.Fatalf("a new SDK plans %v, want %v (edges %v)", s.Reach.Plan.Tiers, want, edges)
 	}
-	if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools", "minio", "umami"}) {
+	if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools"}) {
 		t.Errorf("dependents %v", s.Dependents)
 	}
 }

cmd/mesh-controller/replays_integration_test.go

@@ -316,6 +316,8 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
 	ctx := t.Context()
 	inv := open.inventory
 	asked := asksRecorded(t)
+	// app's key is not built, so the plan stands on its tier as it did while the build ran (see plan_clock_integration_test.go).
+	holdKeys(t, "app")
 	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
 		inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
 			BuiltFrom: "c0", Head: "c0"}); err != nil {
@@ -328,11 +330,12 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
 	}
 	advanceHeld(ctx, open)
 	recent, err := inv.RecentPlans(ctx, 1)
-	if err != nil || len(recent) != 1 || len(*asked) != 1 {
+	if err != nil || len(recent) != 1 || len(*asked) != 0 {
 		t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked)
 	}
 	before := recent[0]
-	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
+	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" ||
+		!before.Modules["app"].Key || before.Modules["app"].KeyAt != "3ad22356aa" {
 		t.Fatalf("the plan is not waiting on its build: %+v", before)
 	}
 	for range 3 {

cmd/mesh-controller/rollout_events_test.go

@@ -31,7 +31,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 
 	// A quarter of an hour of silence is two seconds here.
 	wasSilent := applySilentAfter
-	applySilentAfter = 2 * time.Second
+	// Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's
+	// report never races it: anchor answers tier 1 inside its send (reportOnSend below).
+	applySilentAfter = 10 * time.Second
 	t.Cleanup(func() { applySilentAfter = wasSilent })
 
 	js, err := broker.Dial(testbus.URL(t))
@@ -70,6 +72,8 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	var sends [][]string
 	asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id
 	digests := 0
+	var reportOnSend func(node string) // set below, once a report can be made
+	anchorReported := make(chan error, 1)
 	wasSend := sendRollout
 	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
 		mu.Lock()
@@ -89,6 +93,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 				return nil, err
 			}
 		}
+		// anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by
+		// order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports.
+		if len(sends) == 2 && reportOnSend != nil {
+			for _, node := range names {
+				if node == "anchor" {
+					go reportOnSend(node)
+				}
+			}
+		}
 		return names, nil
 	}
 	t.Cleanup(func() { sendRollout = wasSend })
@@ -147,6 +160,26 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+	// tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal.
+	tell := func(node string) error {
+		last, found, err := inv.SentTo(ctx, node)
+		if err != nil || !found {
+			return fmt.Errorf("%s was sent nothing: %v", node, err)
+		}
+		mu.Lock()
+		seq++
+		n := seq
+		mu.Unlock()
+		body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
+			"report_sequence": n})
+		var report link.Report
+		if err := json.Unmarshal(body, &report); err != nil {
+			return err
+		}
+		_, err = listener.Heard(ctx, report)
+		return err
+	}
+	reportOnSend = func(node string) { anchorReported <- tell(node) }
 	heldKeys := func() map[silentWait]time.Time {
 		c := silentClockOf()
 		if c == nil {
@@ -167,6 +200,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 		return p
 	}
 
+	// Tier 1's module b has its key unbuilt while the first controller runs, so the plan stands at tier 1 as it did while
+	// its build ran; with the key done at once, the same look would take it and send tier 1 as well.
+	heldB := holdKeys(t, "b")
 	// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
 	first, stopFirst := context.WithCancel(ctx)
 	defer stopFirst()
@@ -181,10 +217,14 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
-	eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool {
+	eventually(t, "tier 0 sent to both machines and tier 1 taken from its key, on the save alone", func() bool {
 		mu.Lock()
 		defer mu.Unlock()
-		return len(sends) == 1 && len(*asked) == 1 && plan().Tier == 1
+		// b is taken from its key; the walk carries a commit of novox/mesh-catalog only, and b is registered from
+		// novox/b, so the key is read at b's own trunk value and KeyAt stays empty (Plan.CommitOn names no other
+		// repository's commit).
+		b := plan().Modules["b"]
+		return len(sends) == 1 && len(*asked) == 0 && plan().Tier == 1 && b != nil && b.Key && b.KeyAt == ""
 	})
 	eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 })
 
@@ -194,6 +234,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	time.Sleep(300 * time.Millisecond)
 	reports("anchor")
 
+	heldB.Store(false) // b's key is built by the time the new controller looks
 	second, stopSecond := context.WithCancel(ctx)
 	defer stopSecond()
 	waitSecond := startRolloutEvents(second, open, js)
@@ -212,7 +253,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 		defer mu.Unlock()
 		return len(sends) == 2
 	})
-	reports("anchor")
+	// anchor reported on tier 1 inside its send.
+	select {
+	case err := <-anchorReported:
+		if err != nil {
+			t.Fatalf("anchor's report on tier 1: %v", err)
+		}
+	case <-time.After(15 * time.Second):
+		t.Fatal("anchor never reported on tier 1")
+	}
 
 	// laptop never reports: its key expires and the new controller names it silent, which ends the rollout.
 	eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {
@@ -228,7 +277,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	// No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent
 	// once to both machines, and said as sent once for each.
 	mu.Lock()
-	if len(*asked) != 1 || !strings.HasPrefix((*asked)[0], "novox/mesh-catalog#key-") {
+	if len(*asked) != 1 || !strings.HasPrefix((*asked)[0], "novox/b#key-") { // b is registered from novox/b
 		t.Errorf("taken from keys = %v; want tier 1's module taken once", *asked)
 	}
 	if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" {

cmd/mesh-controller/seatverbs.go

@@ -318,6 +318,8 @@ func (a *verbArguments) commandLine() ([]string, error) {
 		return nil, errors.New("dead-letters is answered by the serving controller, on its own connection, not by a command")
 	case "root-free":
 		return nil, errors.New("root-free is judged by the serving controller, on its own connection, not by a command")
+	case "held":
+		return nil, errors.New("held is asked by the serving controller, of the artifact store, not by a command")
 	case "status":
 		return []string{"status", "--json"}, nil
 	case "nodes":
@@ -1253,6 +1255,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 				if verb == "dead-letters" {
 					return deadLettersAnswer(ctx, a)
 				}
+				if verb == "held" {
+					return heldAnswer(ctx, a.given["references"])
+				}
 				if verb == "root-free" {
 					return rootFreeAnswer(ctx, a.given["machines"], rootClock())
 				}
@@ -1376,7 +1381,10 @@ var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true,
 	"dead-letters": true,
 	// Whether a machine is root-free, judged live on the serving controller's store and connection (novox/hq ADR
 	// 0259 §8): the router asks it before an approval.
-	"root-free": true}
+	"root-free": true,
+	// Whether the artifact store holds what a builder's key names, asked by the builder before it reuses the key (novox/hq
+	// issue 1937).
+	"held": true}
 
 // pushFileVar names the file the controller's push settings are written to (module.json, `push-wait`).
 const pushFileVar = "MESH_PUSH_FILE"

cmd/mesh-controller/silent_mid_tier_integration_test.go

@@ -65,6 +65,9 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
 	var mu sync.Mutex
 	var sends [][]string
 	asksRecorded(t) // each module is taken from a done key as the walk reaches it
+	// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look goes on
+	// through tier 1's send and overwrites the note of the move before the test reads it (see away_integration_test.go).
+	heldB := holdKeys(t, "b")
 	digests := 0
 	wasSend := sendRollout
 	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
@@ -190,7 +193,12 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
 	if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") {
 		t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved)
 	}
-	// Tier 1's module is taken from its key, which the walk's look does by itself.
+	if b := plan().Modules["b"]; b == nil || !b.Key {
+		t.Errorf("tier 1's module is not taken from its key: %+v", b)
+	}
+	// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
+	heldB.Store(false)
+	askToLook()
 	eventually(t, "tier 1 sent to anchor alone", func() bool {
 		mu.Lock()
 		defer mu.Unlock()

cmd/mesh-controller/walk_said_integration_test.go

@@ -3,14 +3,17 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"os"
 	"testing"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/nats-io/nats.go"
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/builderkeys"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 	"github.com/novox/mesh-controller/internal/testbus"
@@ -205,11 +208,17 @@ func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) {
 		if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") {
 			t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery)
 		}
-		// The build asked fails again: its record says so, and the look settles the walk from it.
+		// The retry takes a from its key (it asks no build), at the commit the walk carries.
-		if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: p.Modules["a"].Build, Module: "a", Commit: "c0ffee",
+		if a := p.Modules["a"]; a == nil || !a.Key || a.KeyAt != "c0ffee" {
-			Repository: "novox/a", At: time.Now().UTC(), Asked: before, Failed: "it broke again"}); err != nil {
+			t.Fatalf("%s: the retried walk does not take a from its key: %+v", again[0], a)
-			t.Fatal(err)
 		}
+		// The key fails again: the builder's record says so, and the look settles the walk from it.
+		wholeKeys := openPlanKeys
+		openPlanKeys = func() (planKeyReads, func(), error) {
+			all, done, err := wholeKeys()
+			return keysFailing{all, "a", "it broke again"}, done, err
+		}
+		t.Cleanup(func() { openPlanKeys = wholeKeys })
 		advancePlans(ctx, open)
 		if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed {
 			t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome())
@@ -259,3 +268,18 @@ func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T)
 		t.Fatalf("said again: %v", again)
 	}
 }
+
+// keysFailing is the stand-in builder's keys but for one module, whose key is failed in these words.
+type keysFailing struct {
+	planKeyReads
+	module string
+	words  string
+}
+
+func (k keysFailing) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
+	if module == k.module {
+		return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1,
+			Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: k.words}}, nil
+	}
+	return k.planKeyReads.Build(ctx, module, fp)
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

go.sum

@@ -2,6 +2,8 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYg
 git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/broker/nats.go

@@ -406,6 +406,12 @@ var DegradableGrants = []DegradableGrant{
 		Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
 			"first push, and keeps the packages each node declares only after that, holding the lease (ADR 0507 §5, issue 1840)",
 	},
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".event.artifacts-gone",
+		Reason: "refused: the take-in is refused all the same and says why; only the builder's rebuild waits for the " +
+			"controller's first push, and the builder also asks held when a module's value or its own start comes (novox/hq " +
+			"issue 1937)",
+	},
 	{
 		Subject: "mesh.seat." + ControllerSeat + ".event.sent",
 		Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +

internal/broker/streams.go

@@ -379,7 +379,9 @@ var ControllerStates = []string{"applied", "refused", "built-before",
 	"sent",
 	// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
 	// each registered module's declared config (design 57 F3).
-	"assignment-changed", "declared-config-changed", "predefined-changed"}
+	"assignment-changed", "declared-config-changed", "predefined-changed",
+	// And a builder's key not taken in because the artifact store does not hold what it names (novox/hq issue 1937).
+	"artifacts-gone"}
 
 // BusAdvisories are what the bus server says about the mesh's own account that the controller
 // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it

internal/broker/testdata/composed.conf

@@ -29,7 +29,7 @@ accounts {
     ]
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/inventory/chain_builds.go

@@ -658,9 +658,11 @@ func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id strin
 	if err != nil || !ok {
 		return Build{}, false, err
 	}
+	// **The fingerprint read back with the rest** (novox/hq issue 1938): the controller's grants are keyed by it
+	// (grants.go currentFingerprints), and a build read without it wrote no grant at all from 2026-10-18 on.
 	b := fromChainBuild(rec)
 	return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On,
-		Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil
+		Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At, SourceFingerprint: b.SourceFingerprint}, true, nil
 }
 
 func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) {

internal/inventory/chain_store_test.go

@@ -26,6 +26,7 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
 	first.ReuseKey = "rk-1"
 	second := aBuild("build-2", "notes", "")
 	second.Commit = "c2"
+	second.SourceFingerprint = "src2:" + strings.Repeat("2", 64)
 	failed := aBuild("build-3", "notes", "compile error")
 	other := aBuild("build-4", "wiki", "")
 	for _, b := range []Build{first, second, failed, other, first} {
@@ -48,6 +49,10 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
 	if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
 		t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
 	}
+	// 1938: the fingerprint is read back, which the controller's grants are keyed by.
+	if one.SourceFingerprint != second.SourceFingerprint {
+		t.Errorf("build-2 reads back the fingerprint %q; want %q", one.SourceFingerprint, second.SourceFingerprint)
+	}
 	if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
 		t.Fatal("a build nobody recorded was found")
 	}

internal/link/events.go

@@ -89,6 +89,9 @@ const (
 	KeyDeclaredConfigChanged = "declared-config-changed"
 	// KeyPredefinedChanged: the predefined-keys mirror's one key was written (design 57 K3): a newer list kept.
 	KeyPredefinedChanged = "predefined-changed"
+	// KeyArtifactsGone: a builder's key was not taken in because the artifact store does not hold an artifact it names
+	// (novox/hq issue 1937): the module, the key, its fingerprint and every gone reference. The builder builds the key again.
+	KeyArtifactsGone = "artifacts-gone"
 )
 
 // Applied is what a machine now runs, as the mesh states it.

module.json

@@ -102,6 +102,7 @@
     "data",
     "build",
     "artifacts",
+    "held",
     "collect",
     "images",
     "mirrors",

testdata/beside/CAPTURED

@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
 
 mesh-catalog  2246ccc492e59579a140752e6f574b2121d296af  modules/*/module.json, modules/*/files, modules/*/templates, predefined.json
 mesh-host     9d9e3a46c4f5943cfffb413d962ce48b75e2fd39  examples/foundation-first-node.lock
-mesh-sdk      a2c813a6b468899bb5a95666b8f6f305be52f200  conformance/events/module-event.json
+mesh-sdk      34f210e5eba1d16abe06b526de1026c282692b83  conformance/events/module-event.json
 
 To move them, from this repository's root, with the two repositories checked out beside it: