Hold the expiry of a pending assignment to the words it has said since R4g: its last build, when, and that it failed (issue 1938)

The test was kept from older runs and ran again only once the inventory changed beneath it; e419d018 changed the words.

cmd/mesh-controller/asker_retry_integration_test.go

@@ -23,7 +23,7 @@ import (
 func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
 	open := aMesh(t)
 	ctx := t.Context()
-	asked := asksRecorded(t)
+	asksRecorded(t)
 	planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
 	twoTiers(t, open)
 	before := time.Now().UTC().Add(-time.Hour)
@@ -123,12 +123,9 @@ func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *test
 		t.Fatal(err)
 	}
 	m := p.Modules["a"]
-	if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" {
+	if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" {
 		t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
 	}
-	if len(*asked) != 1 {
-		t.Fatalf("the build was asked %d times: %v", len(*asked), *asked)
-	}
 	if got := store[w.Ask]; got.Acted != "done" {
 		t.Fatalf("the ask ended %q", got.Acted)
 	}

cmd/mesh-controller/away_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -15,6 +16,7 @@ import (
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/builderkeys"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 	"github.com/novox/mesh-controller/internal/testbus"
@@ -68,6 +70,16 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 	var mu sync.Mutex
 	var sends [][]string
 	everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it
+	// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look
+	// goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask
+	// was where the walk paused).
+	wholeKeys := openPlanKeys
+	var holdB atomic.Bool
+	holdB.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{all, "b", &holdB, ""}, done, err
+	}
 	digests := 0
 	wasSend := sendRollout
 	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
@@ -189,7 +201,9 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 	if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
 		t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
 	}
-	// Tier 1's module is taken from its key, which the walk's look does by itself.
+	// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
+	holdB.Store(false)
+	askToLook()
 	eventually(t, "tier 1 sent to anchor alone", func() bool {
 		mu.Lock()
 		defer mu.Unlock()
@@ -211,3 +225,39 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 		t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
 	}
 }
+
+// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of
+// the fingerprint fp when that is given.
+type keysHolding struct {
+	planKeyReads
+	module string
+	held   *atomic.Bool
+	fp     string
+}
+
+func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
+	if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) {
+		return builderkeys.Key{Module: module, Fingerprint: fp}, nil
+	}
+	return k.planKeyReads.Build(ctx, module, fp)
+}
+
+// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared,
+// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of
+// the test, and the flag is set until the test lowers it.
+func holdKeys(t *testing.T, modules ...string) *atomic.Bool {
+	t.Helper()
+	held := &atomic.Bool{}
+	held.Store(true)
+	was := openPlanKeys
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := was()
+		var keys planKeyReads = all
+		for _, m := range modules {
+			keys = keysHolding{keys, m, held, ""}
+		}
+		return keys, done, err
+	}
+	t.Cleanup(func() { openPlanKeys = was })
+	return held
+}

cmd/mesh-controller/backups_before_send_integration_test.go

@@ -211,8 +211,12 @@ func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItA
 	}
 	seats["laptop"].answers("notes", true, "")
 	advancePlans(ctx, open)
-	if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 {
-		t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked)
+	// One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key
+	// build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes
+	// it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key.
+	if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 ||
+		got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" {
+		t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked)
 	}
 	if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
 		t.Fatalf("the condition stays: %+v", said)

cmd/mesh-controller/batches_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"slices"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -65,7 +66,7 @@ func cutAt(t *testing.T, open *stores, now time.Time) {
 func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
 	open := windowed(t)
 	everyHeadWaits(t)
-	asked := asksWithPaths(t)
+	asksWithPaths(t)
 	claude := catalogueMerge("553b7191claude", "app", t0)
 	dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
 	hear(t, open, claude, t0.Add(time.Second))
@@ -96,8 +97,12 @@ func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
 	if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
 		t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
 	}
-	if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" {
-		t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked)
+	// The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at
+	// which commit: the one the branch holds, which is the later merge the walk carries.
+	for _, m := range w.Tiers[w.Tier] {
+		if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit {
+			t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s)
+		}
 	}
 	// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
 	hear(t, open, claude, t0.Add(5*time.Minute))
@@ -131,11 +136,10 @@ func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
 			t.Fatalf("the walk does not build %s: %v", r, w.Modules)
 		}
 	}
-	// Each module's ask is recorded at its own repository's commit.
-	for _, r := range []string{"one", "two", "three"} {
-		q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build)
-		if err != nil || !found || q.Commit != "c-"+r {
-			t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err)
+	// Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build).
+	for _, r := range w.Tiers[w.Tier] {
+		if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r {
+			t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s)
 		}
 	}
 }
@@ -340,7 +344,7 @@ func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
 func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
 	open := windowed(t)
 	everyHeadWaits(t)
-	asked := asksWithPaths(t)
+	asksWithPaths(t)
 	ctx := t.Context()
 	// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
 	oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
@@ -367,9 +371,10 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
 		!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
 		t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
 	}
-	if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app",
-		middle.Commit} {
-		t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked)
+	// The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the
+	// module is taken from its key at the commit the merge was walked on.
+	if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit {
+		t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules)
 	}
 	// Retried, the failed walk would name merges the search answers now.
 	if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
@@ -508,6 +513,14 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
 func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
 	open := windowed(t)
 	asked := asksWithPaths(t)
+	// The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with
+	// the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other.
+	wholeKeys, held := openPlanKeys, &atomic.Bool{}
+	held.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err
+	}
 	ctx := t.Context()
 	hear(t, open, repoMerge("one", "c1", t0), t0)
 	cutAt(t, open, t0.Add(2*time.Minute))
@@ -541,7 +554,11 @@ func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
 			t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
 		}
 	}
-	done := started[0]
+	// Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale.
+	done, err := open.inventory.PlanByID(ctx, started[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
 	done.State = inventory.PlanDone
 	if err := open.inventory.SavePlan(ctx, &done); err != nil {
 		t.Fatal(err)
@@ -624,6 +641,14 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
 	// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
 	// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
 	catalogueWalk := ws[0]
+	// The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the
+	// stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk.
+	wholeKeys, held := openPlanKeys, &atomic.Bool{}
+	held.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err
+	}
 	hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
 	cutAt(t, open, t0.Add(6*time.Minute))
 	ws, _ = walks(t, open)
@@ -642,7 +667,11 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
 	if len(*asked) != before {
 		t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
 	}
-	own2 := ws[0]
+	// Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale.
+	own2, err := open.inventory.PlanByID(ctx, ws[0].ID)
+	if err != nil {
+		t.Fatal(err)
+	}
 	own2.State = inventory.PlanDone
 	if err := open.inventory.SavePlan(ctx, &own2); err != nil {
 		t.Fatal(err)

cmd/mesh-controller/beside_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -15,6 +16,14 @@ import (
 func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
 	open := windowed(t)
 	asksWithPaths(t)
+	// Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in
+	// builder's key done at once, one step would carry a one-module walk to done.
+	wholeKeys, held := openPlanKeys, &atomic.Bool{}
+	held.Store(true)
+	openPlanKeys = func() (planKeyReads, func(), error) {
+		all, done, err := wholeKeys()
+		return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err
+	}
 	ctx := t.Context()
 	hear(t, open, repoMerge("one", "c1", t0), t0)
 	cutAt(t, open, t0.Add(90*time.Second))

cmd/mesh-controller/busobjects_test.go

@@ -46,25 +46,33 @@ func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer {
 	return broker.Consumer{}
 }
 
-// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller
-// started, and the worker its seat's queue should have for it.
+// aLateHolder is a module holding a build seat (node-build-go) on anchor, assigned after the controller started, and
+// the worker its seat's queue should have for it: a build seat is pulled by kind, so the worker is the one for the
+// seat's first accepted kind (the build agent's seat, which had one worker for all, is gone with the builder).
 func aLateHolder(t *testing.T, open *stores) broker.Consumer {
 	t.Helper()
+	// With an account to read, as a holder has: a module with none is no bus user, so its seat's workers are not derived.
 	register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1",
-		Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}})
+		OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/late-builder/broker"}},
+		Claims:     []catalogue.Claim{{Name: "node-build-go", Scope: catalogue.ScopeNode}}})
 	if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil {
 		t.Fatal(err)
 	}
+	_, workers, err := seatTrafficObjects(t.Context(), open.inventory)
+	if err != nil {
+		t.Fatal(err)
+	}
 	for _, s := range inventory.MeshSeats() {
-		if s.Name == "node-build-agent" {
-			c, needed := broker.HolderConsumerFor("anchor", "late-builder", s)
-			if !needed {
-				t.Fatal("the build agent's seat needs no worker")
+		if s.Name != "node-build-go" || len(s.Accepts) == 0 {
+			continue
+		}
+		for _, w := range workers {
+			if w.Name == broker.WorkerName(s.Name, s.Accepts[0]) {
+				return w
 			}
-			return c
 		}
 	}
-	t.Fatal("the mesh declares no build agent's seat")
+	t.Fatal("the build seat's holder is derived no worker")
 	return broker.Consumer{}
 }
 

cmd/mesh-controller/delivery_integration_test.go

@@ -72,7 +72,10 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 	}
 
 	// No holder on record: the merge starts its walk, as before.
+	// A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no
+	// build), so each walk that starts is stepped once before its takes are counted.
 	p := merge("c1aaaaaaaa", "modules/app/index.ts")
+	advanceHeld(ctx, open)
 	if p.Waiting() || len(*asked) != 1 {
 		t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
 	}
@@ -116,6 +119,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 	// ADR 0276): the merge's batch is cut once it ended.
 	finish(p.ID)
 	p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
+	advanceHeld(ctx, open)
 	if p.Waiting() || len(*asked) != 3 {
 		t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
 	}
@@ -157,7 +161,9 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 		t.Fatal(err)
 	}
 	asks := len(*asked)
-	if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
+	p = merge("c6ffffffff", "modules/app/index.ts")
+	advanceHeld(ctx, open)
+	if p.Waiting() || len(*asked) != asks+1 {
 		t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
 	}
 }

cmd/mesh-controller/one_group_one_plan_test.go

@@ -94,8 +94,10 @@ func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *t
 			if plan.Delivery != nil && (plan.Delivery.Awaits != "" || plan.Waiting()) {
 				t.Fatalf("a plan carrying the controller waits for %q's word", plan.Delivery.Awaits)
 			}
-			if plan.Modules["mesh-controller"].Build == "" || len(*asked) != 1 {
-				t.Fatalf("the first tier is asked once: build %q, %d ask(s) %v", plan.Modules["mesh-controller"].Build, len(*asked), *asked)
+			// The cut takes the first tier from the controller's key at its commit and asks no build; the take-in itself is a
+			// later look's, which this test does not run, so nothing is recorded as asked.
+			if c := plan.Modules["mesh-controller"]; !c.Key || c.State != "asked" || c.KeyAt != "k1" || len(*asked) != 0 {
+				t.Fatalf("the first tier is not taken from its key once: %+v, %d ask(s) %v", c, len(*asked), *asked)
 			}
 			// Every merge is answered by that one plan.
 			for _, repository := range []string{"novox/mesh-controller", "novox/mesh-catalog"} {

cmd/mesh-controller/pending_integration_test.go

@@ -151,7 +151,9 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
 	if len(pending) != 1 || pending[0].State != inventory.PendingExpired {
 		t.Fatalf("pending after a failed build: %+v", pending)
 	}
-	for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed",
+	// Its words since R4g (whyNotBuilt): the build, when it was heard, and that it failed (issue 1938: kept from an older
+	// run until the inventory changed beneath it).
+	for _, want := range []string{"expired", "modules/sensors in novox/mesh-catalog; its last build, b-modules/sensors at", ", failed: go build",
 		"undefined: sensorsRead", "nothing was assigned", "build \"true\""} {
 		if !strings.Contains(pending[0].Note, want) {
 			t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note)
@@ -266,9 +268,18 @@ func TestAPlansBuildRequestIsThePlans(t *testing.T) {
 	if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
 		t.Fatal(err)
 	}
+	// A walk asks no build, so it leaves no build request (askModule takes the module from its key): the plan's app is
+	// taken from the key at the commit the plan carries.
 	requests, err := open.inventory.RequestsNamed(ctx, "app")
-	if err != nil || len(requests) != 1 || requests[0].For != "plan" {
-		t.Fatalf("the plan's request: %+v %v", requests, err)
+	if err != nil || len(requests) != 0 {
+		t.Fatalf("a request was recorded for the plan: %+v %v", requests, err)
+	}
+	recent, err := open.inventory.RecentPlans(ctx, 5)
+	if err != nil || len(recent) != 1 {
+		t.Fatalf("the plans: %+v %v", recent, err)
+	}
+	if app := recent[0].Modules["app"]; app == nil || !app.Key || app.KeyAt != "c1aaaaaaaa" {
+		t.Fatalf("the plan's app is not taken from its key at the commit it carries: %+v", app)
 	}
 }
 

cmd/mesh-controller/plan_clock_integration_test.go

@@ -15,6 +15,9 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
 	ctx := t.Context()
 	inv := open.inventory
 	asked := asksRecorded(t)
+	// app's key is not built, so the plan stands on its tier as it did while the build ran; with the stand-in builder's
+	// key done at once, one step would carry it to done.
+	holdKeys(t, "app")
 	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
 		inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
 			BuiltFrom: "c0", Head: "c0"}); err != nil {
@@ -27,11 +30,12 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
 	}
 	advanceHeld(ctx, open)
 	recent, err := inv.RecentPlans(ctx, 1)
-	if err != nil || len(recent) != 1 || len(*asked) != 1 {
+	if err != nil || len(recent) != 1 || len(*asked) != 0 {
 		t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked)
 	}
 	before := recent[0]
-	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
+	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" ||
+		!before.Modules["app"].Key || before.Modules["app"].KeyAt != "c1aaaaaaaa" {
 		t.Fatalf("the plan is not waiting on its build: %+v", before)
 	}
 
@@ -46,7 +50,7 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) {
 		t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s",
 			before.Revision, after.Revision, before.Updated, after.Updated)
 	}
-	if len(*asked) != 1 {
-		t.Fatalf("advancing asked again: %v", *asked)
+	if len(*asked) != 0 {
+		t.Fatalf("advancing took the key in: %v", *asked)
 	}
 }

cmd/mesh-controller/reconnects_integration_test.go

@@ -52,7 +52,9 @@ func TestTheServingControllerLendsItsOwnConnection(t *testing.T) {
 	if a := answer.(verbAnswer); !a.OK || a.Answer == nil {
 		t.Fatalf("hand-acts answered %+v", a)
 	}
-	_, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`))
+	// A read-only verb that runs in the serving process, as a verb that reads the bus does: it uses the
+	// serving connection and opens none. (A verb run as a command re-executes this test binary, which never ends.)
+	_, _ = handlers["dead-letters"](context.Background(), json.RawMessage(`{}`))
 	after, _ := bus.Varz(nil)
 	if opened := after.TotalConnections - before.TotalConnections; opened != 0 {
 		t.Fatalf("the serving controller opened %d connection(s) of its own", opened)

cmd/mesh-controller/replays_integration_test.go

@@ -316,6 +316,8 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
 	ctx := t.Context()
 	inv := open.inventory
 	asked := asksRecorded(t)
+	// app's key is not built, so the plan stands on its tier as it did while the build ran (see plan_clock_integration_test.go).
+	holdKeys(t, "app")
 	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
 		inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
 			BuiltFrom: "c0", Head: "c0"}); err != nil {
@@ -328,11 +330,12 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) {
 	}
 	advanceHeld(ctx, open)
 	recent, err := inv.RecentPlans(ctx, 1)
-	if err != nil || len(recent) != 1 || len(*asked) != 1 {
+	if err != nil || len(recent) != 1 || len(*asked) != 0 {
 		t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked)
 	}
 	before := recent[0]
-	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" {
+	if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" ||
+		!before.Modules["app"].Key || before.Modules["app"].KeyAt != "3ad22356aa" {
 		t.Fatalf("the plan is not waiting on its build: %+v", before)
 	}
 	for range 3 {

cmd/mesh-controller/rollout_events_test.go

@@ -167,6 +167,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 		return p
 	}
 
+	// Tier 1's module b has its key unbuilt while the first controller runs, so the plan stands at tier 1 as it did while
+	// its build ran; with the key done at once, the same look would take it and send tier 1 as well.
+	heldB := holdKeys(t, "b")
 	// The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything.
 	first, stopFirst := context.WithCancel(ctx)
 	defer stopFirst()
@@ -181,10 +184,11 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
-	eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool {
+	eventually(t, "tier 0 sent to both machines and tier 1 taken from its key, on the save alone", func() bool {
 		mu.Lock()
 		defer mu.Unlock()
-		return len(sends) == 1 && len(*asked) == 1 && plan().Tier == 1
+		b := plan().Modules["b"]
+		return len(sends) == 1 && len(*asked) == 0 && plan().Tier == 1 && b != nil && b.Key && b.KeyAt == "c2"
 	})
 	eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 })
 
@@ -194,6 +198,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	time.Sleep(300 * time.Millisecond)
 	reports("anchor")
 
+	heldB.Store(false) // b's key is built by the time the new controller looks
 	second, stopSecond := context.WithCancel(ctx)
 	defer stopSecond()
 	waitSecond := startRolloutEvents(second, open, js)

cmd/mesh-controller/silent_mid_tier_integration_test.go

@@ -65,6 +65,9 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
 	var mu sync.Mutex
 	var sends [][]string
 	asksRecorded(t) // each module is taken from a done key as the walk reaches it
+	// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look goes on
+	// through tier 1's send and overwrites the note of the move before the test reads it (see away_integration_test.go).
+	heldB := holdKeys(t, "b")
 	digests := 0
 	wasSend := sendRollout
 	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
@@ -190,7 +193,12 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
 	if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") {
 		t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved)
 	}
-	// Tier 1's module is taken from its key, which the walk's look does by itself.
+	if b := plan().Modules["b"]; b == nil || !b.Key {
+		t.Errorf("tier 1's module is not taken from its key: %+v", b)
+	}
+	// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
+	heldB.Store(false)
+	askToLook()
 	eventually(t, "tier 1 sent to anchor alone", func() bool {
 		mu.Lock()
 		defer mu.Unlock()

cmd/mesh-controller/walk_said_integration_test.go

@@ -3,14 +3,17 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"os"
 	"testing"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
 	"github.com/nats-io/nats.go"
 	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/builderkeys"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 	"github.com/novox/mesh-controller/internal/testbus"
@@ -205,11 +208,17 @@ func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) {
 		if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") {
 			t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery)
 		}
-		// The build asked fails again: its record says so, and the look settles the walk from it.
-		if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: p.Modules["a"].Build, Module: "a", Commit: "c0ffee",
-			Repository: "novox/a", At: time.Now().UTC(), Asked: before, Failed: "it broke again"}); err != nil {
-			t.Fatal(err)
+		// The retry takes a from its key (it asks no build), at the commit the walk carries.
+		if a := p.Modules["a"]; a == nil || !a.Key || a.KeyAt != "c0ffee" {
+			t.Fatalf("%s: the retried walk does not take a from its key: %+v", again[0], a)
 		}
+		// The key fails again: the builder's record says so, and the look settles the walk from it.
+		wholeKeys := openPlanKeys
+		openPlanKeys = func() (planKeyReads, func(), error) {
+			all, done, err := wholeKeys()
+			return keysFailing{all, "a", "it broke again"}, done, err
+		}
+		t.Cleanup(func() { openPlanKeys = wholeKeys })
 		advancePlans(ctx, open)
 		if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed {
 			t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome())
@@ -259,3 +268,18 @@ func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T)
 		t.Fatalf("said again: %v", again)
 	}
 }
+
+// keysFailing is the stand-in builder's keys but for one module, whose key is failed in these words.
+type keysFailing struct {
+	planKeyReads
+	module string
+	words  string
+}
+
+func (k keysFailing) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
+	if module == k.module {
+		return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1,
+			Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: k.words}}, nil
+	}
+	return k.planKeyReads.Build(ctx, module, fp)
+}

internal/inventory/chain_builds.go

@@ -658,9 +658,11 @@ func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id strin
 	if err != nil || !ok {
 		return Build{}, false, err
 	}
+	// **The fingerprint read back with the rest** (novox/hq issue 1938): the controller's grants are keyed by it
+	// (grants.go currentFingerprints), and a build read without it wrote no grant at all from 2026-10-18 on.
 	b := fromChainBuild(rec)
 	return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On,
-		Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil
+		Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At, SourceFingerprint: b.SourceFingerprint}, true, nil
 }
 
 func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) {

internal/inventory/chain_store_test.go

@@ -26,6 +26,7 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
 	first.ReuseKey = "rk-1"
 	second := aBuild("build-2", "notes", "")
 	second.Commit = "c2"
+	second.SourceFingerprint = "src2:" + strings.Repeat("2", 64)
 	failed := aBuild("build-3", "notes", "compile error")
 	other := aBuild("build-4", "wiki", "")
 	for _, b := range []Build{first, second, failed, other, first} {
@@ -48,6 +49,10 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
 	if err != nil || !found || one.Commit != "c2" || one.Module != "notes" {
 		t.Fatalf("build-2 reads as %+v %v %v", one, found, err)
 	}
+	// 1938: the fingerprint is read back, which the controller's grants are keyed by.
+	if one.SourceFingerprint != second.SourceFingerprint {
+		t.Errorf("build-2 reads back the fingerprint %q; want %q", one.SourceFingerprint, second.SourceFingerprint)
+	}
 	if _, found, _ := inv.BuildByID(ctx, "build-nope"); found {
 		t.Fatal("a build nobody recorded was found")
 	}