Take a walk's modules from the builder's done keys and ask no build but the builder's own: a key building or unreferenced holds the tier and says so, a failed key fails the walk in its words (issue 1851, A6, A7)

cmd/mesh-controller/ask_bus_look_integration_test.go

@@ -31,6 +31,7 @@ func TestALookComesBackForATierTheBusRefusedWhole(t *testing.T) {
 		t.Fatal(err)
 	}
 	bus := false
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
 		if !bus {
@@ -95,6 +96,7 @@ func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) {
 	if err := inv.SavePlan(ctx, &p); err != nil {
 		t.Fatal(err)
 	}
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(context.Context, buildSource, string, string) (string, error) {
 		return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)

cmd/mesh-controller/ask_bus_moment_test.go

@@ -15,6 +15,15 @@ import (
 // seconds) is not a failed build. The module is left unasked and the plan building, and the next look asks it again; only a
 // module's own fault, or a bus silent for ten minutes, fails the plan.
 
+// asksOfEveryModule makes every module a test names the builder's own, the one module a walk still asks build-agent
+// for (ADR 0453 §5): what a test of that ask's waits reads.
+func asksOfEveryModule(t *testing.T) {
+	t.Helper()
+	was := buildersOwn
+	buildersOwn = func(string) bool { return true }
+	t.Cleanup(func() { buildersOwn = was })
+}
+
 func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) {
 	p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b", "c"}},
 		Modules: map[string]*inventory.PlanModule{}}
@@ -26,6 +35,7 @@ func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) {
 }
 
 func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) {
+	asksOfEveryModule(t)
 	p, byName := aTierOfThree()
 	bus := false
 	was := askABuild
@@ -68,6 +78,7 @@ func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) {
 }
 
 func TestABusSilentForTenMinutesFailsTheTierAsBefore(t *testing.T) {
+	asksOfEveryModule(t)
 	p, byName := aTierOfThree()
 	was := askABuild
 	askABuild = func(context.Context, buildSource, string, string) (string, error) {
@@ -119,6 +130,7 @@ func TestAPlanWithAModuleWaitingForTheBusIsOneTheLookComesBackTo(t *testing.T) {
 }
 
 func TestAModulesOwnFaultStillFailsThePlan(t *testing.T) {
+	asksOfEveryModule(t)
 	p, byName := aTierOfThree()
 	delete(byName, "a")
 	askModule(t.Context(), p, "a", byName)

cmd/mesh-controller/away_integration_test.go

@@ -68,6 +68,7 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
 	var mu sync.Mutex
 	var asks []string
 	var sends [][]string
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	wasAsk := askABuild
 	askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
 		mu.Lock()

cmd/mesh-controller/builder_on_the_bus_test.go

@@ -117,6 +117,7 @@ func noBuildAsked(t *testing.T) {
 // the test's own may ask at once, so the record is written under a lock (issue 1543).
 func asksWithPaths(t *testing.T) *[][3]string {
 	t.Helper()
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it
 	var asked [][3]string
 	var mu sync.Mutex
 	was := askABuild

cmd/mesh-controller/pending_integration_test.go

@@ -305,6 +305,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
 	open := aCatalogueMesh(t)
 	ctx := t.Context()
 	inv := open.inventory
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(context.Context, buildSource, string, string) (string, error) {
 		return "", errors.New("cannot submit a build: nats: timeout")
@@ -569,6 +570,7 @@ func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) {
 	open := aCatalogueMesh(t)
 	ctx := t.Context()
 	asked := 0
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(context.Context, buildSource, string, string) (string, error) {
 		asked++

cmd/mesh-controller/plan_keys.go

@@ -0,0 +1,268 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6
+// and A7).
+//
+// From R4e a walk asks build-agent for no module but the builder's own (ADR 0453 §5). A module of its tier is marked
+// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value
+// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every
+// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done
+// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words.
+// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules).
+
+// isBuildersOwn says a module is the builder's own: the one module build-agent still builds, asked on a merge, until R4g
+// (ADR 0453 §5, ADR 0498 §7).
+func isBuildersOwn(module string) bool { return buildersOwn(module) }
+
+// buildersOwn is isBuildersOwn's rule: a variable only so a test of the one ask left (its bus waits, its plan key) may
+// name modules of its own as the builder's.
+var buildersOwn = func(module string) bool { return module == sdkdelivery.BuilderModule }
+
+// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's).
+type planKeyReads interface {
+	ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error)
+	Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
+}
+
+// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus.
+var openPlanKeys = func() (planKeyReads, func(), error) {
+	r, closeKeys, err := openBuilderKeys()
+	if err != nil {
+		return nil, nil, err
+	}
+	return r, closeKeys, nil
+}
+
+// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6).
+func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) {
+	state.BusWaitFrom = nil
+	state.State = "asked"
+	state.AskedAt = &now
+	state.Key = true
+	state.Build, state.Fingerprint = "", ""
+	state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))
+	state.Why = "waits for the builder's key"
+	state.Commit, state.BuiltAt, state.Made = "", nil, ""
+	fmt.Printf("  tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module)
+}
+
+// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it
+// waits: kept in this process, so a look starts each once and a settle says why it waits.
+var keyTakes = struct {
+	sync.Mutex
+	running map[string]bool
+	waits   map[string]string
+	// taken are the keys whose take-in ended registered, or found the module already at a later build (superseded).
+	taken map[string]bool
+}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}}
+
+// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's
+// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only
+// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look.
+func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) {
+	return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) {
+		go takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id)
+	})
+}
+
+// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's).
+type keyRecords interface {
+	BuildByID(ctx context.Context, id string) (inventory.Build, bool, error)
+	RefusedOf(ctx context.Context, ids []string) (map[string]string, error)
+	SourceOf(ctx context.Context, module string) (inventory.Source, error)
+}
+
+// settleKeys is settleFromKeys on the records handed in, starting a take-in with take.
+func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string,
+	take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) {
+	changed := false
+	set := func(s *inventory.PlanModule, why string) {
+		if s.Why != why {
+			s.Why = why
+			changed = true
+		}
+	}
+	for _, m := range tier {
+		s := p.Modules[m]
+		if s == nil || !s.Key || s.State != "asked" {
+			continue
+		}
+		trunk, err := keys.ModuleAt(ctx, m, s.KeyAt)
+		if errors.Is(err, builderkeys.ErrNoValue) {
+			set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt)))
+			continue
+		}
+		if err != nil {
+			return changed, err
+		}
+		if trunk.FP2 == "" {
+			why := trunk.None2
+			if why == "" {
+				why = "the indexer wrote no fingerprint of the new form"
+			}
+			set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why))
+			continue
+		}
+		if s.Fingerprint != trunk.FP2 {
+			s.Fingerprint, changed = trunk.FP2, true
+		}
+		k, err := keys.Build(ctx, m, trunk.FP2)
+		if err != nil {
+			return changed, err
+		}
+		switch {
+		case k.Failed():
+			// **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`,
+			// `version taken:`), and a rebuild of it is a new revision the next walk takes.
+			s.State, s.Why = "failed", k.Build.Error
+			p.State = inventory.PlanFailed
+			p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error))
+			changed = true
+			continue
+		case !k.Done() || len(k.Unreferenced()) > 0:
+			set(s, "waits: "+k.Says())
+			continue
+		}
+		id := keyBuildID(k)
+		refused, err := inv.RefusedOf(ctx, []string{id})
+		if err != nil {
+			return changed, fmt.Errorf("the refused registrations cannot be read: %w", err)
+		}
+		for kept, why := range refusalsKeptFor([]string{id}) {
+			refused[kept] = why
+		}
+		if why, ok := refused[id]; ok {
+			s.State, s.Why, s.Build = "failed", why, id
+			p.State = inventory.PlanFailed
+			p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why))
+			changed = true
+			continue
+		}
+		b, found, err := inv.BuildByID(ctx, id)
+		if err != nil {
+			return changed, err
+		}
+		if found && b.Worked() {
+			keyTakes.Lock()
+			taken := keyTakes.taken[id]
+			keyTakes.Unlock()
+			registered := taken
+			if !registered {
+				if registered, err = registeredFrom(ctx, inv, m, b); err != nil {
+					return changed, err
+				}
+			}
+			if registered {
+				at := b.At
+				s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, b.Commit, id
+				s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision)
+				s.Why = ""
+				fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2))
+				changed = true
+				continue
+			}
+		}
+		keyTakes.Lock()
+		running, waits := keyTakes.running[id], keyTakes.waits[id]
+		if !running {
+			keyTakes.running[id] = true
+		}
+		keyTakes.Unlock()
+		if waits != "" {
+			set(s, firstLine(waits))
+		} else {
+			set(s, "taking in the builder's done key "+short(trunk.FP2))
+		}
+		if !running {
+			take(m, trunk, id)
+		}
+	}
+	return changed, nil
+}
+
+// registeredFrom says the module's registration stands on the build: the catalogue holds the module at the build's
+// commit, or the build was registered as one with an unchanged source (no move).
+func registeredFrom(ctx context.Context, inv keyRecords, module string, b inventory.Build) (bool, error) {
+	src, err := inv.SourceOf(ctx, module)
+	if errors.Is(err, inventory.ErrNoSuchModule) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return sameCommitID(src.BuiltFrom, b.Commit), nil
+}
+
+func sameCommitID(a, b string) bool {
+	if a == "" || b == "" {
+		return false
+	}
+	n := min(len(a), len(b))
+	return n >= 7 && a[:n] == b[:n]
+}
+
+func orNow(commit string) string {
+	if commit == "" {
+		return "its value now"
+	}
+	return short(commit)
+}
+
+// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the
+// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the
+// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look.
+func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) {
+	defer func() {
+		keyTakes.Lock()
+		delete(keyTakes.running, id)
+		keyTakes.Unlock()
+		askToLook()
+	}()
+	keys, closeKeys, err := openBuilderKeys()
+	if err != nil {
+		keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error())
+		return
+	}
+	defer closeKeys()
+	_, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk,
+		Ask: registrationAsk{Merge: trunk.Commit}})
+	switch {
+	case err == nil, errors.Is(err, inventory.ErrSuperseded):
+		// Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries
+		// this merge is answered by, as a build-agent outcome superseded was.
+		keepKeyWait(id, "")
+		keyTakes.Lock()
+		keyTakes.taken[id] = true
+		keyTakes.Unlock()
+	case errors.Is(err, errKeyWaits):
+		keepKeyWait(id, err.Error())
+	default:
+		why := err.Error()
+		if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil {
+			fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr)
+		}
+		keepRefusalFor(id, why, time.Now())
+	}
+}
+
+func keepKeyWait(id, why string) {
+	keyTakes.Lock()
+	defer keyTakes.Unlock()
+	if why == "" {
+		delete(keyTakes.waits, id)
+		return
+	}
+	keyTakes.waits[id] = why
+}

cmd/mesh-controller/plan_keys_test.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/novox/mesh-controller/internal/builderkeys"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by
+// module.
+type planKeysOf struct {
+	values map[string]sdkdelivery.Module // by module + "@" + commit
+	keys   map[string]builderkeys.Key
+}
+
+func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) {
+	if v, ok := k.values[module+"@"+commit]; ok {
+		return v, nil
+	}
+	return sdkdelivery.Module{}, builderkeys.ErrNoValue
+}
+
+func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
+	if key, ok := k.keys[module]; ok && key.Fingerprint == fp {
+		return key, nil
+	}
+	return builderkeys.Key{Module: module, Fingerprint: fp}, nil
+}
+
+// recordsOf is the build records and refusals a settle reads, for a test.
+type recordsOf struct {
+	builds  map[string]inventory.Build
+	refused map[string]string
+	sources map[string]inventory.Source
+}
+
+func (r recordsOf) BuildByID(_ context.Context, id string) (inventory.Build, bool, error) {
+	b, ok := r.builds[id]
+	return b, ok, nil
+}
+
+func (r recordsOf) RefusedOf(_ context.Context, ids []string) (map[string]string, error) {
+	out := map[string]string{}
+	for _, id := range ids {
+		if why, ok := r.refused[id]; ok {
+			out[id] = why
+		}
+	}
+	return out, nil
+}
+
+func (r recordsOf) SourceOf(_ context.Context, module string) (inventory.Source, error) {
+	if s, ok := r.sources[module]; ok {
+		return s, nil
+	}
+	return inventory.Source{}, inventory.ErrNoSuchModule
+}
+
+func aWalkOfOne(module string) (*inventory.Plan, map[string]inventory.Entry) {
+	p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{module}},
+		Modules: map[string]*inventory.PlanModule{}, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789",
+		Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789"}}}}
+	byName := map[string]inventory.Entry{module: {Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
+		Path: "modules/" + module, Ref: "main"}}}
+	return p, byName
+}
+
+// A6 (ADR 0503 §6, ADR 0498 check §7): a walk whose module has a done key goes asked to built with no build asked: the
+// ask stub is never called, the key is taken in once, and `plans` says it was taken from the builder's key.
+func TestAWalkTakesADoneKeyAndAsksNoBuild(t *testing.T) {
+	was := askABuild
+	askABuild = func(context.Context, buildSource, string, string) (string, error) {
+		t.Fatal("a walk asked build-agent for a module that is not the builder's own")
+		return "", nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	p, byName := aWalkOfOne("sudo")
+	askModule(t.Context(), p, "sudo", byName)
+	s := p.Modules["sudo"]
+	if s.State != "asked" || !s.Key || s.Build != "" || s.KeyAt != "c0ffee0123456789" {
+		t.Fatalf("the module is not taken from a key: %+v", s)
+	}
+	fp := "src2:" + strings.Repeat("a", 64)
+	k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 2, Build: sdkdelivery.BuilderBuild{
+		State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a",
+			Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}}
+	keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}},
+		keys: map[string]builderkeys.Key{"sudo": k}}
+	records := recordsOf{builds: map[string]inventory.Build{}, sources: map[string]inventory.Source{}}
+	var taken []string
+	take := func(m string, _ sdkdelivery.Module, id string) { taken = append(taken, m+" "+id) }
+	t.Cleanup(func() {
+		keyTakes.Lock()
+		keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
+		keyTakes.Unlock()
+	})
+
+	changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take)
+	if err != nil || !changed || len(taken) != 1 || s.State != "asked" || s.Fingerprint != fp {
+		t.Fatalf("a done key not recorded yet is taken in once: %v %v %v %+v", changed, err, taken, s)
+	}
+	// A second look while the take-in runs starts no second one.
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || len(taken) != 1 {
+		t.Fatalf("a second look took the key in again: %v %v", taken, err)
+	}
+	// The take-in recorded and registered it: the look settles it built from the record.
+	id := keyBuildID(k)
+	records.builds[id] = inventory.Build{ID: id, Module: "sudo", Commit: "c0ffee0123456789", At: time.Now()}
+	records.sources["sudo"] = inventory.Source{BuiltFrom: "c0ffee0123456789"}
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil {
+		t.Fatal(err)
+	}
+	if s.State != "built" || s.Build != id || !strings.Contains(s.Made, "taken from the builder's key") {
+		t.Fatalf("the module is not built from its key: %+v", s)
+	}
+	if takesOutcome(*p, &inventory.PlanModule{Key: true, State: "asked", AskedAt: s.AskedAt}, time.Now(), "build-1") {
+		t.Fatal("a build-agent outcome answers a module taken from a key")
+	}
+}
+
+// A7: a key building waits and says so, then moves on when done, with no timer and no ask; a failed key fails the walk
+// in the builder's words; a value the indexer has not written yet waits, named.
+func TestAWalkWaitsForAKeyAndFailsWithItsWords(t *testing.T) {
+	asksOfNone := askABuild
+	askABuild = func(context.Context, buildSource, string, string) (string, error) {
+		t.Fatal("a walk asked a build")
+		return "", nil
+	}
+	t.Cleanup(func() { askABuild = asksOfNone })
+	t.Cleanup(func() {
+		keyTakes.Lock()
+		keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
+		keyTakes.Unlock()
+	})
+	p, byName := aWalkOfOne("view")
+	askModule(t.Context(), p, "view", byName)
+	s := p.Modules["view"]
+	records := recordsOf{builds: map[string]inventory.Build{}}
+	take := func(string, sdkdelivery.Module, string) {}
+	keys := planKeysOf{values: map[string]sdkdelivery.Module{}, keys: map[string]builderkeys.Key{}}
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil ||
+		!strings.Contains(s.Why, "waits for the indexer's value of view at c0ffee01") {
+		t.Fatalf("no indexer value yet: %q %v", s.Why, err)
+	}
+	fp := "src2:" + strings.Repeat("c", 64)
+	keys.values["view@c0ffee0123456789"] = sdkdelivery.Module{FP2: fp}
+	keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true,
+		Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}}
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil ||
+		s.State != "asked" || !strings.Contains(s.Why, "building on g14") || p.State != inventory.PlanBuilding {
+		t.Fatalf("a building key: %+v %v", s, err)
+	}
+	keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true,
+		Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: the store answered 500"}}
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil {
+		t.Fatal(err)
+	}
+	if s.State != "failed" || p.State != inventory.PlanFailed || !strings.Contains(p.Note, "not stored: tools") {
+		t.Fatalf("a failed key does not fail the walk in its words: %+v %q", s, p.Note)
+	}
+}
+
+// A7: a take-in refused (a module from another repository, a manifest refused) fails the walk with the refusal.
+func TestARefusedTakeInFailsTheWalk(t *testing.T) {
+	t.Cleanup(func() {
+		keyTakes.Lock()
+		keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
+		keyTakes.Unlock()
+	})
+	p, byName := aWalkOfOne("sudo")
+	askModule(t.Context(), p, "sudo", byName)
+	fp := "src2:" + strings.Repeat("d", 64)
+	k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{
+		State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a",
+			Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}}
+	keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
+		keys: map[string]builderkeys.Key{"sudo": k}}
+	records := recordsOf{refused: map[string]string{keyBuildID(k): "sudo is built from novox/mesh-catalog, and this build is of agent/sudo"}}
+	if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil {
+		t.Fatal(err)
+	}
+	if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "agent/sudo") {
+		t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note)
+	}
+}

cmd/mesh-controller/plan_retry.go

@@ -402,7 +402,9 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	var asked []string
 	for _, m := range failed {
 		askModule(ctx, &p, m, byName)
-		if s := p.Modules[m]; s.State == "asked" {
+		if s := p.Modules[m]; s.State == "asked" && s.Key {
+			asked = append(asked, m+" from the builder's key")
+		} else if s.State == "asked" {
 			asked = append(asked, m+" as "+s.Build)
 		}
 	}
@@ -629,7 +631,9 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
 	for _, m := range again {
 		sendAgain(p.Modules[m])
 		askModule(ctx, p, m, byName)
-		if s := p.Modules[m]; s.State == "asked" {
+		if s := p.Modules[m]; s.State == "asked" && s.Key {
+			asked = append(asked, m+" from the builder's key")
+		} else if s.State == "asked" {
 			asked = append(asked, m+" as "+s.Build)
 		}
 	}

cmd/mesh-controller/plans_building_test.go

@@ -156,6 +156,7 @@ func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
 
 // The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
 func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
+	asksOfEveryModule(t)
 	store := newFakeBuildingStore()
 	var s buildingStore = store
 	buildingKeys.Store(&s)

cmd/mesh-controller/queue_integration_test.go

@@ -431,6 +431,7 @@ func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
 	asked := asksRecorded(t)
 	twoTiers(t, open)
 	var refs []string
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
 		refs = append(refs, ref)

cmd/mesh-controller/release_plan.go

@@ -390,6 +390,11 @@ func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time
 	if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) {
 		return false
 	}
+	// **A module taken from the builder's key is answered by its key alone** (ADR 0501 §1): a build-agent outcome of it
+	// is somebody else's build.
+	if state.Key {
+		return false
+	}
 	if state.Build != "" {
 		return state.Build == id
 	}
@@ -565,6 +570,13 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
 		p.Note = name + " is no longer in the catalogue"
 		return
 	}
+	// **Every module but the builder's own is taken from the builder's done key, asking no build** (novox/hq ADR 0503
+	// §6, ADR 0501, ADR 0453 §5): the look settles it (settleFromKeys).
+	if !isBuildersOwn(name) {
+		sayBuilding(ctx, p.ID, now)
+		takeFromKey(p, state, e, now)
+		return
+	}
 	source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
 	fmt.Printf("  tier %d: ", p.Tier)
 	// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every
@@ -1071,6 +1083,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// the controller in its first tier: the build that produced the new one is recorded, and the
 	// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
 	// outcome, whoever was listening.
+	// **Taken from the builder's keys first** (ADR 0503 §6, issue 1851): every module of the tier but the builder's own.
+	if keyed(p, tier) {
+		keys, closeKeys, err := openPlanKeys()
+		if err != nil {
+			return false, err
+		}
+		was := p.State
+		changed, err := settleFromKeys(ctx, open, keys, p, tier)
+		closeKeys()
+		if err != nil {
+			return false, err
+		}
+		if p.State == inventory.PlanFailed && was != inventory.PlanFailed {
+			sayUnsent(p, rollsOut)
+		}
+		if changed {
+			return true, nil
+		}
+	}
 	recorded, byID, err := recordsOfAsked(ctx, inv, p, tier)
 	if err != nil {
 		return false, err
@@ -2744,7 +2775,7 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 	changed := false
 	for _, m := range tier {
 		s := p.Modules[m]
-		if s == nil || s.State != "asked" || s.AskedAt == nil {
+		if s == nil || s.State != "asked" || s.AskedAt == nil || s.Key {
 			continue
 		}
 		var outcome *inventory.Build
@@ -2845,3 +2876,13 @@ func stopByHand(p *inventory.Plan, how, by, why string) {
 	}
 	p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
 }
+
+// keyed says a module of the tier still waits on the builder's key.
+func keyed(p *inventory.Plan, tier []string) bool {
+	for _, m := range tier {
+		if s := p.Modules[m]; s != nil && s.Key && s.State == "asked" {
+			return true
+		}
+	}
+	return false
+}

cmd/mesh-controller/replays_integration_test.go

@@ -423,6 +423,7 @@ func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) {
 	ctx := t.Context()
 	inv := open.inventory
 	var asked [][3]string
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
 		asked = append(asked, [3]string{source.Repository, path, ref})

cmd/mesh-controller/rollout_events_test.go

@@ -69,6 +69,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	var mu sync.Mutex
 	var asks []string
 	var sends [][]string
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	wasAsk := askABuild
 	askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
 		mu.Lock()

cmd/mesh-controller/silent_mid_tier_integration_test.go

@@ -65,6 +65,7 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
 	var mu sync.Mutex
 	var asks []string
 	var sends [][]string
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	wasAsk := askABuild
 	askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
 		mu.Lock()

cmd/mesh-controller/tier_send_integration_test.go

@@ -302,6 +302,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
 		}
 	}
 
+	asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
 	was := askABuild
 	askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil }
 	t.Cleanup(func() { askABuild = was })

cmd/mesh-controller/walk_from_keys_integration_test.go

@@ -0,0 +1,104 @@
+//go:build integration
+
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A6, A7 whole against the store and a real bus (ADR 0503 §6, ADR 0498 check §7; issue 1851): a walk whose modules the
+// builder built goes from asked to built with no build asked, its keys taken in off the plans' hold, then sends its tier
+// to the first machine as before; a key still building holds the tier, said, and moves on when the builder's key is done.
+func TestAWalkTakesItsTierFromTheBuildersKeysAndSendsIt(t *testing.T) {
+	withTheGate(t)
+	judgeLeavesAsTheCore(t)
+	tm := aTierMesh(t, "app1", "app2")
+	ctx := t.Context()
+	inv := tm.open.inventory
+	noBuildAsked(t)
+	b := aBuilderOnTheBus(t)
+
+	// The walk carries c3 of the catalogue; its tier is unasked.
+	p := tm.plan(t)
+	p.Commit = "c3c3c3c3c3c3c3c3"
+	for _, m := range []string{"app1", "app2"} {
+		p.Modules[m] = &inventory.PlanModule{}
+	}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	fp1, fp2 := fpOf('1'), fpOf('2')
+	m1 := b.indexed(t, "app1", "c3", "novox/mesh-catalog", "modules/app1", p.Commit, fp1)
+	m2 := b.indexed(t, "app2", "c3", "novox/mesh-catalog", "modules/app2", p.Commit, fp2)
+	b.built(t, "app1", "c3", fp1, m1, sdkdelivery.BuildDone)
+	b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildBuilding)
+
+	advancePlans(ctx, tm.open)
+	becomes(t, 10*time.Second, "app1 taken from its key", func() bool {
+		advancePlans(ctx, tm.open)
+		return tm.plan(t).Modules["app1"].State == "built"
+	})
+	p = tm.plan(t)
+	if s := p.Modules["app2"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "building on ace") {
+		t.Fatalf("A7: app2, whose key is building, is %+v", s)
+	}
+	if len(tm.sent) != 0 {
+		t.Fatalf("A7: the tier was sent while app2 waits on its key: %v", tm.sent)
+	}
+	if s := p.Modules["app1"]; !strings.Contains(s.Made, "taken from the builder's key") || s.Fingerprint != fp1 {
+		t.Fatalf("A6: plans does not say app1 was taken from the builder's key: %+v", s)
+	}
+
+	b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildDone)
+	becomes(t, 10*time.Second, "the tier sent to the first machine", func() bool {
+		advancePlans(ctx, tm.open)
+		return len(tm.sent) > 0
+	})
+	p = tm.plan(t)
+	if !reflect.DeepEqual(tm.sent[0], []string{"anchor"}) {
+		t.Fatalf("A6: the tier taken from keys was sent %v, not to the first machine", tm.sent)
+	}
+	current, err := inv.CurrentBuilds(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range []string{"app1", "app2"} {
+		if current[m].Commit != p.Commit || !isKeyBuildID(p.Modules[m].Build) {
+			t.Fatalf("A6: %s stands on %+v (%s), not the key's build at %s", m, current[m], p.Modules[m].Build, p.Commit)
+		}
+	}
+}
+
+// A7: a failed key fails the walk with the builder's words, and nothing is sent.
+func TestAFailedKeyFailsTheWalkInTheBuildersWords(t *testing.T) {
+	tm := aTierMesh(t, "app1")
+	ctx := t.Context()
+	inv := tm.open.inventory
+	noBuildAsked(t)
+	b := aBuilderOnTheBus(t)
+	p := tm.plan(t)
+	p.Commit = "c4c4c4c4c4c4c4c4"
+	p.Modules["app1"] = &inventory.PlanModule{}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	fp := fpOf('4')
+	manifest := b.indexed(t, "app1", "c4", "novox/mesh-catalog", "modules/app1", p.Commit, fp)
+	b.built(t, "app1", "c4", fp, manifest, sdkdelivery.BuildFailed)
+	for i := 0; i < 3; i++ {
+		advancePlans(ctx, tm.open)
+	}
+	p = tm.plan(t)
+	if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "digests differ: ace sha256:1, g14 sha256:2") {
+		t.Fatalf("A7: the walk is %s: %q", p.State, p.Note)
+	}
+	if len(tm.sent) != 0 {
+		t.Fatalf("A7: a failed walk sent %v", tm.sent)
+	}
+}

internal/inventory/plans.go

@@ -388,6 +388,15 @@ type PlanModule struct {
 	// data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it
 	// could not be judged.
 	Recorded map[string]string `json:"recorded,omitempty"`
+	// Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453
+	// §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome.
+	Key bool `json:"key,omitempty"`
+	// KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty
+	// for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now.
+	KeyAt string `json:"key_at,omitempty"`
+	// Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value;
+	// empty until the indexer has written it.
+	Fingerprint string `json:"fingerprint,omitempty"`
 }
 
 // PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,