mesh-delivery/modules/mesh-delivery-builder/internal/build/run_test.go

package build

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"slices"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-delivery/sdk/forms"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"

	"mesh-delivery-builder/internal/versions"
)

// buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the
// step's end into the steps (as mesh-build-go and mesh-build-node do), with a digest by node and artifact, or a failure.
type buildModules struct {
	mu      sync.Mutex
	n       int
	calls   []call
	steps   *fakeSteps
	digests map[string]string // `<node>/<artifact>` -> digest; the default is `digest`
	fails   map[string]string // `<node>/<tool>` -> the reason the step fails with
	// noRef names `<node>/<artifact>` whose bundle or archive build answers no reference (its push not answered).
	noRef map[string]bool
	// packages are the package artifacts by name, each with the `<name>@<version>` its trunk build publishes (ADR 0498 clause 5).
	packages map[string]string
	// hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives.
	// conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict.
	conflicts string
	noMerge   bool
	hold      chan struct{}
	entered   chan struct{}
}

func newBuildModules() *buildModules {
	return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}}
}

func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) {
	if b.hold != nil {
		select {
		case b.entered <- struct{}{}:
		default:
		}
		<-b.hold
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	b.calls = append(b.calls, call{node, key, body})
	b.n++
	id := fmt.Sprintf("s%d", b.n)
	module, tool, _ := strings.Cut(key, ".")
	args, _ := body.(map[string]any)
	step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepPassed}
	if reason, ok := b.fails[node+"/"+tool]; ok {
		step.State, step.Reason = delivery.StepFailed, reason
	} else if strings.HasSuffix(tool, "_fetch") {
		step.Result = json.RawMessage(`{"workspace":"w-` + node + `"}`)
		if args["base"] != nil { // a fetch with a base answers the merge, or the files in conflict (mesh-build-go FetchResult)
			if b.conflicts != "" {
				step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","conflicts":["` + b.conflicts + `"]}`)
			} else if !b.noMerge {
				step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","merge":"` + strings.Repeat("9", 40) + `"}`)
			}
		}
	} else {
		d := digest
		if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok {
			d = v
		}
		if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4)
			step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`)
		} else if pkg, ok := b.packages[fmt.Sprint(args["artifact"])]; ok { // a package names its version only when published
			step.Result = json.RawMessage(`{"digest":"` + d + `"}`)
			if args["publish"] == true {
				step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"` + pkg + `"}`)
			}
		} else if tool == "build_image_build" && args["publish"] == true {
			step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`)
		} else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] {
			// A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1).
			step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`)
		} else {
			step.Result = json.RawMessage(`{"digest":"` + d + `"}`)
		}
	}
	sk, _ := delivery.StepKey(node, id)
	b.steps.mu.Lock()
	b.steps.steps[module+"/"+sk] = step
	b.steps.mu.Unlock()
	return json.RawMessage(`{"step":"` + id + `","key":"` + sk + `","state":"running"}`), nil
}

func (b *buildModules) callsOf(tool string) (out []call) {
	b.mu.Lock()
	defer b.mu.Unlock()
	for _, c := range b.calls {
		if strings.HasSuffix(c.key, "."+tool) {
			out = append(out, c)
		}
	}
	return out
}

type fakeManifests struct {
	byDigest map[string]catalogue.Manifest
}

func (f *fakeManifests) Manifest(_ context.Context, d string) (catalogue.Manifest, error) {
	m, ok := f.byDigest[d]
	if !ok {
		return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", d)
	}
	return m, nil
}

func mdigest(c byte) string { return hex64(c) }

type rig struct {
	b     *Builder
	st    *memStore
	bm    *buildModules
	tools *buildrun.ToolchainSet
	mf    *fakeManifests
	key   string
	// asks are the build seats' streams; puller is the node that pulls every ask ("ace" when empty).
	asks   *buildrun.FakeAsks
	puller string
}

// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}

// answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the
// rig's pulling node, and answers the step that node writes.
func (r *rig) answer(a delivery.StepAsk) delivery.Step {
	node := r.puller
	if node == "" {
		node = "ace"
	}
	module, _ := delivery.BuildModuleOf(a.Language)
	runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}
	tool := func(name string) stepFunc {
		return func(ctx context.Context, args map[string]any) (any, error) {
			raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args)
			if err != nil {
				return nil, err
			}
			return raw, nil
		}
	}
	steps := askSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
	out, err := runAsk(context.Background(), steps, a)
	step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed}
	if err != nil {
		step.State, step.Reason = delivery.StepFailed, err.Error()
		return step
	}
	step.Result, _ = json.Marshal(out)
	return step
}

func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
	t.Helper()
	r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
	for _, n := range []string{"ace", "novox"} {
		r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
		r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
	}
	r.asks = &buildrun.FakeAsks{Answer: r.answer}
	r.b = &Builder{
		Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
		Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
		Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
	}
	var err error
	if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
		t.Fatal(err)
	}
	return r
}

// agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names.
var agreedToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}

func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) }

func moduleValueWith(t *testing.T, path string, toolchains map[string]delivery.ToolchainUsed) []byte {
	t.Helper()
	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: toolchains})
	if err != nil {
		t.Fatal(err)
	}
	return data
}

func (r *rig) consider(t *testing.T, path string) delivery.BuilderBuild {
	t.Helper()
	r.b.Module(context.Background(), "m", moduleValue(t, path))
	r.b.Wait()
	if _, ok := r.st.vals[r.key]; !ok {
		t.Fatal("no build key was written")
	}
	return valueOf(t, r.st, r.key)
}

var (
	goBundle  = catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}
	tsBundle  = catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"}
	archiveA  = catalogue.Artifact{Name: "files", Kind: "archive"}
	imageArt  = catalogue.Artifact{Name: "img", Kind: "image"}
	manifestD = mdigest('a')
)

func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
	t.Run("the imports a build answers reach the done key and the log", importsABuildAnswersReachTheDoneKeyAndTheLog)
	t.Run("the out-of-step files a build answers reach the log", outOfStepFilesABuildAnswersReachTheLog)
	t.Run("ADR 0624: a done key keeps each form a counted node reads", aDoneKeyKeepsEachFormACountedNodeReads)
	r := newRig(t, goBundle, archiveA, tsBundle)
	// ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out.
	r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false)
	r.tools.Put(buildrun.NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc4"), false)
	defer func() {
		if st := r.st.vals[r.key]; !strings.Contains(string(st), "ace sits out: typescript tsc4, agreed tsc5") {
			t.Errorf("the key does not say ace sits out: %s", st)
		}
	}()
	var logged []delivery.BuildLogEntry
	var lmu sync.Mutex
	r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
	v := r.consider(t, "modules/m")
	// 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read
	// from the two.
	if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent ||
		logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" ||
		logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" ||
		logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" ||
		logged[0].FP != fp1 || logged[0].Module != "m" {
		t.Fatalf("logged = %+v", logged)
	}
	for _, e := range logged {
		if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
			t.Errorf("an entry the log refuses: %v", err)
		}
	}
	if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok {
		t.Error("no duration from the two entries")
	}
	// builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more.
	var page stdio.LogPage
	for i, e := range logged {
		raw, _ := delivery.EncodeBuildLogEntry(e)
		page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At})
	}
	page.Next = 3
	since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At)
	if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More {
		t.Errorf("builds since = %+v, %v, want the one end and more", since, err)
	}
	page.Entries[0].Value = []byte(`{"v":0}`)
	if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil {
		t.Error("an entry that does not read was skipped, want it refused")
	}
	if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
		t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v)
	}
	for i, want := range []string{"files", "tool", "web"} {
		if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "artifact-store://m/"+want+"/blobs/"+digest {
			t.Errorf("artifact %d = %+v, want %s with the digest and its artifact-store reference (ADR 0498 clause 1)", i, a, want)
		}
	}
	if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" {
		t.Errorf("kinds = %+v", v.Artifacts)
	}
	// ADR 0618 §1: the done key names the forms its references are written in, here the first form alone, and keeps no list
	// beside the single reference while no counted node needs another form.
	if !slices.Equal(v.Needs, []string{"form:artifact-store"}) || v.Artifacts[0].References != nil {
		t.Errorf("needs = %v, references = %v", v.Needs, v.Artifacts[0].References)
	}
	if got := v.Toolchains["go"]; got.Version != "go1.27.1" || got.Recipe != "1" {
		t.Errorf("toolchains = %+v, want go and typescript named", v.Toolchains)
	}
	if got := v.Toolchains["typescript"]; got.Version != "tsc5" {
		t.Errorf("toolchains = %+v", v.Toolchains)
	}
	// One fetch per build module, on one node (ADR 0478 clause 4); every build names the module's directory.
	if n := len(r.bm.callsOf("build_go_fetch")); n != 1 {
		t.Errorf("%d go fetches, want one", n)
	}
	if n := len(r.bm.callsOf("build_go_build")); n != 2 { // tool and files
		t.Errorf("%d go builds, want 2", n)
	}
	if n := len(r.bm.callsOf("build_node_build")); n != 1 {
		t.Errorf("%d node builds, want 1", n)
	}
	c := r.bm.callsOf("build_go_build")[0]
	if args := c.body.(map[string]any); args["module"] != "modules/m" || !strings.HasPrefix(fmt.Sprint(args["workspace"]), "w-") {
		t.Errorf("build args = %+v, want the module's path and the fetch's workspace", args)
	}
	f := r.bm.callsOf("build_go_fetch")[0].body.(map[string]any)
	if f["repository"] != "novox/mesh-catalog" || f["commit"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" {
		t.Errorf("fetch args = %+v", f)
	}
}

// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
// before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference.
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
	r := newRig(t, goBundle)
	r.asks.Answer = nil // no node has pulled it yet
	// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
	sdk := delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
		FP2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains, Packages: sdk})
	if err != nil {
		t.Fatal(err)
	}
	r.b.Module(context.Background(), "m", data)
	var asks []delivery.StepAsk
	for deadline := time.Now().Add(10 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatal("no ask was put")
		}
		asks = r.asks.Asks()
	}
	a := asks[0]
	if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 {
		t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks)
	}
	if !reflect.DeepEqual(a.Packages, sdk) {
		t.Errorf("the ask's packages = %v, want %v", a.Packages, sdk)
	}
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID {
		t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID)
	}
	if n := len(r.bm.callsOf("build_go_build")) + len(r.bm.callsOf("build_go_fetch")); n != 0 {
		t.Fatalf("the builder called a node %d times", n)
	}
	r.asks.End(r.answer(a)) // a node pulled it and ran it
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || !strings.Contains(v.Step, "on ace") || v.Artifacts[0].Reference == "" || len(v.Asks) != 0 {
		t.Fatalf("the key = %+v, want done on ace naming the reference", v)
	}
}

// 1887/AC3: a builder that starts again while its ask waits puts no second ask: it waits for the step of the ask the
// building key names.
func TestARestartedBuilderWaitsForTheAskTheKeyNames(t *testing.T) {
	r := newRig(t, goBundle)
	const id = "mvl0frv18d1957"
	data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
		Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}})
	r.st.vals[r.key] = data
	r.asks.Answer = nil
	r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed,
		Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)})
	v := r.consider(t, "modules/m")
	if n := len(r.asks.Asks()); n != 0 {
		t.Fatalf("%d asks put again", n)
	}
	if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on shanks") {
		t.Fatalf("the key = %+v, want done from the ask's step on shanks", v)
	}
}

func TestAFailingStepFailsTheBuildWithItsReasonAndNodeAndIsTriedAgainAsANewRevision(t *testing.T) { // 1607/AC7, ADR 0429 clause 18
	r := newRig(t, goBundle)
	r.bm.fails["ace/build_go_build"] = "go build of tool: undefined: x"
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "undefined: x") {
		t.Fatalf("build = %+v, want failed with the node and the reason", v)
	}
	rev := r.st.revs[r.key]
	delete(r.bm.fails, "ace/build_go_build")
	v = r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev {
		t.Errorf("build = %+v at revision %d (was %d), want done as a new revision of the same key", v, r.st.revs[r.key], rev)
	}
}

func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { // 1705a, ADR 0487 clause 5
	r := newRig(t, imageArt)
	r.puller = "ace" // two nodes qualify; an image still builds on one
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest {
		t.Fatalf("build = %+v, want done with one image artifact and its digest", v)
	}
	if v.Artifacts[0].Reference == "" {
		t.Errorf("a trunk image has no reference: %+v", v.Artifacts[0])
	}
	if n := len(r.bm.callsOf("build_image_fetch")); n != 1 {
		t.Errorf("%d image fetches, want one", n)
	}
	writes := r.st.writes
	r.consider(t, "modules/m")
	if r.st.writes != writes || len(r.bm.callsOf("build_image_build")) != 1 {
		t.Errorf("a done image key was built again")
	}
}

func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
	r := newRig(t, goBundle, imageArt)
	r.puller = "ace"
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || len(v.Artifacts) != 2 {
		t.Fatalf("build = %+v", v)
	}
	if n := len(r.bm.callsOf("build_go_build")); n != 1 {
		t.Errorf("%d go builds, want 1 (one node per fingerprint, ADR 0478 clause 4)", n)
	}
	if n := len(r.bm.callsOf("build_image_build")); n != 1 {
		t.Errorf("%d image builds, want 1", n)
	}
}

func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision
	r := newRig(t, imageArt)
	r.puller = "ace"
	trunkModule(t, r)
	// DLV-B6: a check's build of a head a kept tag ask names is numbered under that tag at its claim.
	nums := &fakeNumbers{}
	var tagAsked []string
	r.b.Numbers = nums
	r.b.Tags = func(_ context.Context, repository, head string) (string, bool) {
		tagAsked = append(tagAsked, repository+"@"+head)
		return "feat-x", head == headX
	}
	r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	r.b.Wait()
	if !slices.Equal(nums.tags, []string{"feat-x"}) || len(tagAsked) != 1 || !strings.HasSuffix(tagAsked[0], "@"+headX) {
		t.Errorf("the check's build numbered with tags %v (asked %v), want feat-x for its head", nums.tags, tagAsked)
	}
	v := valueOf(t, r.st, r.key)
	if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" {
		t.Fatalf("build = %+v, want done with no reference", v)
	}
	calls := r.bm.callsOf("build_image_build")
	if len(calls) != 1 {
		t.Fatalf("%d image builds, want 1", len(calls))
	}
	if _, has := calls[0].body.(map[string]any)["publish"]; has {
		t.Errorf("a candidate's image call carries publish: %+v", calls[0].body)
	}
}

func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The versions*: numbered at the claim
	r := newRig(t, imageArt)
	r.puller = "ace"
	nums := &fakeNumbers{wait: "m/img: the package waits for its floor"}
	r.b.Numbers = nums
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if _, ok := r.st.vals[r.key]; ok || nums.numbered != 1 {
		t.Fatalf("a build waiting for its floor was claimed (numbered %d times)", nums.numbered)
	}
	nums.wait, nums.versions = "", map[string]string{"img": "0.1.4"}
	r.consider(t, "modules/m")
	calls := r.bm.callsOf("build_image_build")
	if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true {
		t.Errorf("calls = %+v, want publish true", calls)
	}
	if nums.built["img"] != digest {
		t.Errorf("the built digest recorded = %v, want %s for img", nums.built, digest)
	}
	if nums.done != nil {
		t.Errorf("a build still to make was numbered with done digests %v", nums.done)
	}
	// The indexer's packages state names a package of the module (PackageNamed): the done build is looked at again and
	// numbered with its key's digests, so version.numbered carries them for the late publish; nothing is built again.
	looks := nums.numbered
	r.b.PackageNamed(context.Background(), "m")
	r.b.Wait()
	if nums.numbered != looks+1 || nums.done["img"] != digest || len(r.bm.callsOf("build_image_build")) != 1 {
		t.Errorf("a named package's done build: numbered %d more times with %v, %d image builds; want once with img %s and no build",
			nums.numbered-looks, nums.done, len(r.bm.callsOf("build_image_build")), digest)
	}
	r.b.PackageNamed(context.Background(), "other")
	r.b.Wait()
	if nums.numbered != looks+1 {
		t.Errorf("a module the builder does not know was looked at")
	}
}

// fakeNumbers is Numbers that answers what it is set to and keeps the digests a done build recorded.
type fakeNumbers struct {
	wait     string
	versions map[string]string
	numbered int
	built    map[string]string
	tags     []string
	done     map[string]string // the done key's digests the last look numbered with
}

func (f *fakeNumbers) Number(_ context.Context, _ string, _ delivery.Module, _ Plan, tag string, done map[string]string) (versions.Numbering, error) {
	f.tags = append(f.tags, tag)
	f.done = done
	f.numbered++
	return versions.Numbering{Wait: f.wait, Versions: f.versions}, nil
}

func (f *fakeNumbers) Built(_ context.Context, _ string, _ delivery.Module, _ Plan, got versions.Numbering, made []delivery.BuiltArtifact) error {
	f.built = map[string]string{}
	for _, a := range made {
		if got.Versions[a.Name] != "" {
			f.built[a.Name] = a.Digest
		}
	}
	return nil
}

func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4
	r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"})
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotPinned) || !strings.Contains(v.Error, "up") {
		t.Fatalf("build = %+v, want failed 'not pinned:' naming the artifact", v)
	}
	if len(r.bm.calls) != 0 {
		t.Errorf("%d calls, want none for an unpinned upstream", len(r.bm.calls))
	}
	writes := r.st.writes
	r.consider(t, "modules/m")
	if r.st.writes != writes {
		t.Errorf("an unpinned upstream's key was written again (%d writes, was %d)", r.st.writes, writes)
	}
}

func TestAModuleKeyWithNoPathFailsNamingThatReason(t *testing.T) { // 1607/AC8, ADR 0478 clause 8
	r := newRig(t, goBundle)
	v := r.consider(t, "")
	if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "path") || len(r.bm.calls) != 0 {
		t.Errorf("build = %+v with %d calls, want failed naming the missing path and nothing called", v, len(r.bm.calls))
	}
}

func TestAnUnreadableManifestFailsTheBuild(t *testing.T) {
	r := newRig(t, goBundle)
	delete(r.mf.byDigest, mdigest('a'))
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "manifest") {
		t.Errorf("build = %+v, want failed naming the manifest", v)
	}
}

func TestAModuleWithNoFingerprintIsNotBuilt(t *testing.T) { // 1607/AC2
	r := newRig(t)
	data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"})
	r.b.Module(context.Background(), "n", data)
	r.b.Wait()
	if len(r.st.vals) != 0 {
		t.Errorf("keys %v were written: a module with no new-form fingerprint has nothing to build", r.st.vals)
	}
}

func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testing.T) { // issue 1613, ADR 0478 clause 1
	for name, manifest := range map[string]catalogue.Manifest{"no build": {Module: "m"}, "no artifact": {Module: "m", Build: &catalogue.Build{}}} {
		r := newRig(t)
		r.mf.byDigest[mdigest('a')] = manifest
		v := r.consider(t, "") // a module with nothing to build needs no directory
		if v.State != delivery.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" {
			t.Errorf("%s: build = %+v, want done with no artifact", name, v)
		}
		if len(r.bm.calls) != 0 {
			t.Errorf("%s: %d build module calls, want none", name, len(r.bm.calls))
		}
		writes := r.st.writes
		r.b.tried = map[string]bool{}
		r.consider(t, "")
		if r.st.writes != writes {
			t.Errorf("%s: the done key was written again", name)
		}
	}
}

func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *testing.T) { // 1607/AC3, AC7
	r := newRig(t, goBundle)
	seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD})
	if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone {
		t.Fatalf("build = %+v, want the builder's own building key built", v)
	}
	calls := len(r.bm.calls)
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Reconsider(context.Background())
	r.b.Wait()
	if len(r.bm.calls) != calls {
		t.Errorf("a done key was built again (%d calls, was %d)", len(r.bm.calls), calls)
	}
}

// Two instances that start together on one key: the second finds the key building and takes it over (ADR 0478 clause 6:
// a key found building at start is taken over, whoever holds it), so both may build; the key is written once, by the one
// whose compare-and-set comes first, and the other's write is refused, never overwriting it.
func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4
	r := newRig(t, goBundle)
	other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Asks: r.b.Asks, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf}
	value := moduleValue(t, "modules/m")
	var wg sync.WaitGroup
	for _, b := range []*Builder{r.b, other} {
		wg.Add(1)
		go func() { defer wg.Done(); b.Module(context.Background(), "m", value); b.Wait() }()
	}
	wg.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
		t.Errorf("build = %+v, want one done key", v)
	}
	if n := len(r.bm.callsOf("build_go_build")); n < 1 {
		t.Errorf("%d builds, want one for each instance that built", n)
	}
}

func TestAModulesValueThatArrivesWhileItsKeyIsBeingBuiltIsLookedAtAgainWhenItEnds(t *testing.T) { // 1607/AC6 (no event is dropped, no timer)
	r := newRig(t, goBundle)
	r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
	r.bm.hold, r.bm.entered = make(chan struct{}), make(chan struct{}, 8)
	value := moduleValue(t, "modules/m")
	r.b.Module(context.Background(), "m", value)
	<-r.bm.entered // the first build is under way
	r.b.Module(context.Background(), "m", value)
	r.b.Reconsider(context.Background())
	delete(r.bm.fails, "ace/build_go_build") // what the event is about: the build can succeed now
	close(r.bm.hold)
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
		t.Errorf("build = %+v, want the failed key built again as a new revision after the first build ended", v)
	}
}

func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4
	r := newRig(t, goBundle)
	old := map[string]delivery.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}}
	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
	r.b.Wait()
	if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
		t.Run("a recipe that moves builds the module again under its new fingerprint", aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint)
		t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
	}
	// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}}))
	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
	r.b.Wait()
	if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
		t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls))
	}
	// The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds.
	r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false)
	r.tools.Put(buildrun.GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false)
	r.b.Reconsider(context.Background())
	r.b.Wait()
	if len(r.st.vals) != 0 {
		t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals)
	}
	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}}))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Toolchains["go"].Version != "go1.27.2" {
		t.Errorf("build = %+v, want done with go1.27.2", v)
	}
}

// ADR 0603 §3 (D9): a source revision changing the Go build module's recipe and a Go module builds the module at once with
// the recipe the nodes run and again, a second key under the new fingerprint, once the agreement moves to the new recipe;
// the first key is neither rebuilt nor waited on.
func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) {
	r := newRig(t, goBundle)
	r.consider(t, "modules/m")
	calls := len(r.bm.calls)
	fp2 := "src2:" + hex64('2')
	for _, n := range []string{"ace", "novox"} {
		b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now()})
		if err != nil {
			t.Fatal(err)
		}
		r.tools.Put(buildrun.GoBuildModule, n+".go", b, false)
	}
	r.b.Reconsider(context.Background())
	r.b.Wait()
	if len(r.bm.calls) != calls {
		t.Errorf("%d calls, was %d: the first fingerprint was built again for the new recipe", len(r.bm.calls), calls)
	}
	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp2,
		Manifest: mdigest('a'), Path: "modules/m", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}}})
	if err != nil {
		t.Fatal(err)
	}
	r.b.Module(context.Background(), "m", data)
	r.b.Wait()
	second, err := delivery.BuilderBuildKey("m", fp2)
	if err != nil {
		t.Fatal(err)
	}
	if v := valueOf(t, r.st, second); v.State != delivery.BuildDone || v.Toolchains["go"].Recipe != "2" {
		t.Errorf("second build = %+v, want done with recipe 2", v)
	}
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
		t.Errorf("first build = %+v, want it left done", v)
	}
}

func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
	r := newRig(t, goBundle)
	r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
		t.Errorf("keys %v and %d calls while the nodes disagree", r.st.vals, len(r.bm.calls))
	}
}

func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/AC4
	r := newRig(t, goBundle)
	if v := r.consider(t, delivery.RootPath); v.State != delivery.BuildDone {
		t.Fatalf("build = %+v, want done for a root module", v)
	}
	if args := r.bm.callsOf("build_go_build")[0].body.(map[string]any); args["module"] != "." {
		t.Errorf("build args = %+v, want module \".\"", args)
	}
}

func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5
	r := newRig(t, imageArt)
	r.puller = "ace"
	trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's
	r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	r.b.Wait()
	v := valueOf(t, r.st, r.key)
	if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "" {
		t.Fatalf("candidate build = %+v, want done with no reference", v)
	}
	if n := len(r.bm.callsOf("build_image_build")); n != 1 {
		t.Fatalf("%d image builds after the candidate, want 1", n)
	}
	// The pull request is merged: the trunk's record names the candidate's fingerprint.
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	v = valueOf(t, r.st, r.key)
	if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" {
		t.Fatalf("trunk build = %+v, want done with a reference", v)
	}
	calls := r.bm.callsOf("build_image_build")
	published := 0
	for _, c := range calls {
		if c.body.(map[string]any)["publish"] == true {
			published++
		}
	}
	if len(calls) != 2 || published != 1 {
		t.Errorf("%d image builds, %d with publish, want 2 and exactly 1", len(calls), published)
	}
	// Written once more only: looking again builds nothing.
	rev := r.st.revs[r.key]
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_image_build")) != 2 {
		t.Error("a published key was built again")
	}
}

func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5
	r := newRig(t, imageArt)
	r.puller = "ace"
	trunkModule(t, r)
	r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
		t.Fatalf("candidate build = %+v", v)
	}
	r.bm.fails["ace/build_image_build"] = "push to the registry: connection refused"
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk names the fingerprint
	r.b.Wait()
	v := valueOf(t, r.st, r.key)
	if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") {
		t.Fatalf("build = %+v, want failed naming the publish and its reason", v)
	}
	// The failed key is retaken as any failed key is, and ends published once the registry answers.
	delete(r.bm.fails, "ace/build_image_build")
	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
	r.b.Wait()
	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Artifacts[0].Reference == "" {
		t.Errorf("build = %+v, want done with a reference", v)
	}
}

// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
// drop from their stream (Dropper).
type goneAsks struct {
	mu      sync.Mutex
	n       int
	why     func(a delivery.StepAsk) string
	dropped []string
}

func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
	g.mu.Lock()
	defer g.mu.Unlock()
	g.dropped = append(g.dropped, a.ID)
	return nil
}

func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
	g.mu.Lock()
	g.n++
	g.mu.Unlock()
	return g.why(a), nil
}

func (g *goneAsks) asked() int {
	g.mu.Lock()
	defer g.mu.Unlock()
	return g.n
}

// waitForAsks waits until n asks were put.
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
	t.Helper()
	for deadline := time.Now().Add(10 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
		if time.Now().After(deadline) {
			t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
		}
	}
	return r.asks.Asks()
}

// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
	t.Run("waits in its stream: never asked again", func(t *testing.T) {
		r := newRig(t, goBundle)
		r.asks.Answer = nil
		r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
		gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
		r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
		r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
		waitForAsks(t, r, 1)
		// Five check bounds passed (five reads of whether it can be answered), so the ask waited past the step bound: an event
		// to wait on, where a pause of 100 ms said nothing on a loaded machine.
		for deadline := time.Now().Add(10 * time.Second); gone.asked() < 5; time.Sleep(time.Millisecond) {
			if time.Now().After(deadline) {
				t.Fatalf("whether the ask can still be answered was read %d times within 10 s, want 5", gone.asked())
			}
		}
		if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
			t.Fatalf("the key = %+v while its ask waits, want building", v)
		}
		if gone.asked() == 0 {
			t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
		}
		r.asks.End(r.answer(r.asks.Asks()[0]))
		r.b.Wait()
		if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
			t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
		}
	})
	t.Run("declined by a node off the agreed toolchain: put again, no asking used, the bound ends it loudly", func(t *testing.T) { // ADR 0584 clause 2
		r := newRig(t, goBundle)
		r.asks.Answer = nil
		// The builder's clock steps a second at each read, so the bound of a minute ends the declines after many asks however
		// slow the machine (a bound of 150 ms of wall clock let a loaded one put fewer than DefaultAskings).
		var clock atomic.Int64
		start := time.Now()
		r.b.Now = func() time.Time { return start.Add(time.Duration(clock.Add(1)) * time.Second) }
		r.b.DeclinePause, r.b.DeclineBound = -1, time.Minute
		r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
		declined, _ := json.Marshal(delivery.StepDeclined{Declined: true, Lacks: []string{"the agreed toolchain go1.27.1 (recipe 1)"}})
		waitForAsks(t, r, 1)
		for ended, deadline := 0, time.Now().Add(20*time.Second); valueOf(t, r.st, r.key).State == delivery.BuildBuilding && time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
			if asks := r.asks.Asks(); len(asks) > ended { // each ask put again is declined in its turn
				r.asks.End(delivery.Step{ID: asks[ended].ID, Node: "ace", State: delivery.StepRefused, Reason: "this node builds with go1.26.0", Result: declined})
				ended++
			}
		}
		r.b.Wait()
		v := valueOf(t, r.st, r.key)
		if n := len(r.asks.Asks()); v.State != delivery.BuildFailed || n <= DefaultAskings || !strings.Contains(v.Error, "declined for") || !strings.Contains(v.Error, "ace: this node builds with go1.26.0") {
			t.Fatalf("the key = %+v after %d asks, want it failed after more than %d asks, naming ace and its reason", v, n, DefaultAskings)
		}
	})
	for _, c := range []struct {
		name    string
		askings int
		// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
		step   string
		byHand bool
		always bool // Gone says every ask cannot be answered, not only the first
		dead   bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
		deaf   bool // the ask's own watch hears nothing though its step ends (the lost check of 2026-10-28)
		puts   int
		done   bool
		asking int // the asking the event says, 0 when none is said
		// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
		// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
		// is refused, "" when it is built again as a new revision and its ask id answered.
		again  bool
		key    *delivery.BuilderBuild
		known  string
		refuse string
		// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
		// the error the key then holds when its build ends before its asks are put.
		edit   func(m *delivery.Module)
		busy   int
		failed string
		// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
		during bool
	}{
		{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
			edit: func(m *delivery.Module) {
				m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
			},
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
			edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
			key:  &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
		{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
			key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
		{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
			key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
		{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
		{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
		{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
			key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
		{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
		{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
		{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
		{name: "its step ended while its watch was deaf: read at its build module's next step end, done with no new ask", deaf: true, puts: 1, done: true},
		{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
		{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
	} {
		t.Run(c.name, func(t *testing.T) { // 1945
			r := newRig(t, goBundle)
			if c.again {
				var rev uint64
				if c.key != nil {
					c.key.Manifest = manifestD
					rev = putBuild(t, r.st, r.key, *c.key)
				}
				m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
				if err != nil {
					t.Fatal(err)
				}
				if c.known != "" {
					m.FP2 = c.known
				}
				if c.edit != nil {
					c.edit(&m)
				}
				r.b.init()
				var stored atomic.Int64
				if c.during {
					r.asks.Answer = nil
					r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
				}
				switch c.busy {
				case 1:
					r.b.inflight[r.key] = true
				case 2:
					r.b.hands[r.key] = &handWatch{}
				}
				r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
				log := &logged{}
				r.b.Logf = log.logf
				res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
				if c.during {
					// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
					r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
					r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
					r.b.Wait()
					if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
						t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
					}
					return
				}
				r.b.Wait()
				after := r.st.revs[r.key]
				if c.refuse != "" {
					if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
						t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
					}
					return
				}
				if c.failed != "" {
					if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
						t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
					}
					return
				}
				asks := r.asks.Asks()
				if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
					t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
				}
				if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
					t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
				}
				if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
					t.Errorf("the why is not logged: %v", log.lines)
				}
				return
			}
			r.asks.Answer = nil
			r.b.Askings = c.askings
			var said []AskedAgain
			var mu sync.Mutex
			r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
			first := ""
			if c.step != "" {
				first = "mvl0frv18d1960"
				data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
					Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
				r.st.vals[r.key] = data
			}
			gone := &goneAsks{}
			dead := &buildrun.DeadLetters{}
			if c.dead {
				r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
			}
			var heard *buildrun.HeardEnds
			if c.deaf {
				heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
				r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
			}
			if !c.byHand && !c.dead && !c.deaf {
				gone.why = func(a delivery.StepAsk) string {
					if c.always || a.ID == first || first == "" {
						return "it waits first in line while a node took a later ask"
					}
					return ""
				}
				r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
			}
			r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
			if c.dead {
				dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
					At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
			}
			if c.byHand {
				waitForAsks(t, r, 1)
				for deadline := time.Now().Add(10 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
					if time.Now().After(deadline) {
						t.Fatal("no lane of the key waits")
					}
				}
			}
			if c.deaf {
				r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
				for deadline := time.Now().Add(10 * time.Second); heard.Waiting() == 0; time.Sleep(time.Millisecond) {
					if time.Now().After(deadline) {
						t.Fatal("no ask is waited on")
					}
				}
				heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
			}
			if c.done && !c.deaf {
				asks := waitForAsks(t, r, c.puts)
				if first == "" && !c.byHand {
					first = asks[0].ID
				}
				last := asks[len(asks)-1]
				if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
					t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
				}
				r.asks.End(r.answer(last))
			}
			r.b.Wait()
			v := valueOf(t, r.st, r.key)
			if n := len(r.asks.Asks()); n != c.puts {
				t.Errorf("%d asks put, want %d", n, c.puts)
			}
			if c.done && v.State != delivery.BuildDone {
				t.Errorf("the key = %+v, want done from the ask asked again", v)
			}
			if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
				t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
			}
			mu.Lock()
			defer mu.Unlock()
			// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
			if gone.mu.Lock(); c.done && !c.byHand && !c.dead && !c.deaf && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
				t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
			}
			gone.mu.Unlock()
			if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
				t.Errorf("said %+v, want once at asking %d", said, c.asking)
			}
			if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
				t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
			}
		})
	}
	t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
		put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
		taken := func(kind string, at time.Time) *Taken {
			return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
		}
		later := taken(delivery.AskDelivery, put.Add(time.Minute))
		// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
		ahead := func(n int, step string) []Queued {
			var qs []Queued
			for i := range n {
				at := put.Add(-time.Duration(n-i) * time.Minute)
				qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
			}
			return qs
		}
		seen := func(qs []Queued, last *Taken) AskSeen {
			return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
		}
		for _, c := range []struct {
			name    string
			seen    AskSeen
			want    string // a part of the why, "" when it can be answered
			waiting int
		}{
			{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
			{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
			{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
			{"first in line, nothing taken since", seen(nil, nil), "", 0},
			{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
			{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
			{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
			// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
			{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
			{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
			{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
			{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
		} {
			got := c.seen.Why()
			if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
				t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
			}
		}
		if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
			t.Errorf("AskedAt does not read an ask id's time")
		}
	})
	t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
		at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
		state := func(vals map[string]string) StateRead {
			return StateRead{Keys: func() ([]string, error) {
				var ks []string
				for k := range vals {
					ks = append(ks, k)
				}
				return ks, nil
			}, Get: func(k string) (json.RawMessage, time.Time, error) {
				if v, ok := vals[k]; ok {
					return json.RawMessage(v), at, nil
				}
				return nil, time.Time{}, nil
			}}
		}
		key, _ := delivery.BuilderBuildKey("m", fp1)
		done, _ := delivery.BuilderBuildKey("d", fp1)
		building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
			Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
		finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
		running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
		_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
		r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
			Steps:   map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
			Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
			Now:     func() time.Time { return at.Add(time.Hour) }}
		out, err := r.Stuck("")
		if err != nil {
			t.Fatal(err)
		}
		// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
		if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
			t.Errorf("dead_letters = %v, want not heard, saying why", dl)
		}
		stuck := out["stuck"].([]Stuck)
		if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
			t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
		}
		a := stuck[0].Asks
		if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
			a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
			t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
		}

		// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
		// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
		// only for the rows kept.
		var gets, written, inFlight, maxFlight atomic.Int64
		vals := map[string]string{}
		for i := 0; i < 3000; i++ {
			k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
			vals[k] = string(finished)
			if (i/100)%10 == 0 {
				vals[k] = string(building)
			}
		}
		slow := state(vals)
		get := slow.Get
		slow.Get = func(k string) (json.RawMessage, time.Time, error) {
			gets.Add(1)
			n := inFlight.Add(1)
			for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
			}
			time.Sleep(time.Millisecond)
			inFlight.Add(-1)
			raw, _, err := get(k)
			return raw, time.Time{}, err
		}
		slow.Written = func(string) time.Time { written.Add(1); return at }
		for _, row := range []struct {
			module string
			stuck  int
			gets   int64
			cut    bool
		}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
			gets.Store(0)
			written.Store(0)
			out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
			if err != nil {
				t.Fatal(err)
			}
			_, cut := out["cut"]
			if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
				t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
			}
		}
		if maxFlight.Load() < 2 {
			t.Errorf("the values were read one at a time")
		}
	})
}

// heldAsks says an ask is held when its id is among held.
type heldAsks map[string]bool

func (h heldAsks) Held(_ context.Context, a delivery.StepAsk) (bool, error) { return h[a.ID], nil }

// 1910: a builder that stopped between naming the ask in its key and putting it left a key naming an ask neither the
// stream holds nor any node ran: started again, the builder puts that ask, under the same id, once, and the key is
// built from its step; an ask the stream holds is not put again.
func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) {
	for _, held := range []bool{false, true} {
		r := newRig(t, goBundle)
		const id = "mvl0frv18d1958"
		data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
			Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}})
		r.st.vals[r.key] = data
		r.b.Held = heldAsks{id: held}
		if held {
			r.asks.Answer = nil
			r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed,
				Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)})
		}
		v := r.consider(t, "modules/m")
		asks := r.asks.Asks()
		switch {
		case held && len(asks) != 0:
			t.Errorf("an ask the stream holds was put again: %+v", asks)
		case !held && (len(asks) != 1 || asks[0].ID != id):
			t.Errorf("the ask nobody put: asks %+v, want one under %s", asks, id)
		}
		if v.State != delivery.BuildDone {
			t.Errorf("held %v: the key = %+v, want done", held, v)
		}
	}
}

// unreadableHeld cannot tell whether an ask is held.
type unreadableHeld struct{}

func (unreadableHeld) Held(context.Context, delivery.StepAsk) (bool, error) {
	return false, errors.New("permissions violation for publish to $JS.API.STREAM.INFO.SEAT_NODE_BUILD_GO")
}

// 1910: a take-over that cannot read whether the ask its key names is on its stream says so and waits for the ask's step:
// the key is not failed, and the build finishes when a node ran the ask.
func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
	r := newRig(t, goBundle)
	const id = "mvl0frv18d1959"
	data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
		Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}})
	r.st.vals[r.key] = data
	r.b.Held = unreadableHeld{}
	r.asks.Answer = nil
	r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed,
		Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)})
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || len(r.asks.Asks()) != 0 {
		t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
	}
}

// silentWatch is a watch of an ask's step that stays open and hears nothing.
type silentWatch struct{}

func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
	return make(chan delivery.Step), nil
}

// aDoneKeyKeepsEachFormACountedNodeReads holds the builder's rows of ADR 0624's check: every build keeps each form a
// counted node's engine reads, present or away, a node stating nothing reading the baseline's first form; the oldest kept
// form is the single reference; a retired node leaves the count. old is the first form, new a fixture second form read
// only by an engine stating form:new.
func aDoneKeyKeepsEachFormACountedNodeReads(t *testing.T) {
	fs := forms.ArtifactForms{
		{Name: "old", First: true, Kinds: []string{"bundle"}, Is: func(r string) bool { return strings.HasPrefix(r, "artifact-store://") }},
		{Name: "new", Kinds: []string{"bundle"}, Is: func(r string) bool { return strings.HasPrefix(r, "10.0.0.1:5000/") }},
	}
	both, nothing, onlyNew := []string{"form:new", "word:module"}, []string(nil), []string{"form:new"}
	oldRef, newRef := "artifact-store://m/code/blobs/"+digest, "10.0.0.1:5000/m/code@"+digest
	for _, row := range []struct {
		name             string
		present, counted map[string][]string
		kept             []string
		reference        string
		references       []string
	}{
		{"A present {old,new}, B away {old}: both forms, the old in the single reference",
			map[string][]string{"a": both}, map[string][]string{"a": both, "b": {"word:module"}}, []string{"old", "new"}, oldRef, []string{oldRef, newRef}},
		{"A present {new}, B away stating nothing: the baseline's form stays",
			map[string][]string{"a": onlyNew}, map[string][]string{"a": onlyNew, "b": nothing}, []string{"old", "new"}, oldRef, []string{oldRef, newRef}},
		{"A present {old,new}, B away and never asked (no key): the baseline's form stays",
			map[string][]string{"a": both}, map[string][]string{"a": both, "b": nil}, []string{"old", "new"}, oldRef, []string{oldRef, newRef}},
		{"B retired: the new form only", map[string][]string{"a": both}, map[string][]string{"a": both}, []string{"new"}, newRef, nil},
		{"B present stating nothing: the new form is not written",
			map[string][]string{"a": both, "b": nothing}, map[string][]string{"a": both, "b": nothing}, []string{"old"}, oldRef, nil},
		{"no node present: the first form", nil, map[string][]string{"a": both}, []string{"old"}, oldRef, nil},
	} {
		if got := fs.Kept("bundle", row.present, row.counted); !slices.Equal(got, row.kept) {
			t.Errorf("%s: kept %v, want %v", row.name, got, row.kept)
		}
		a := fs.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{newRef, oldRef}, row.present, row.counted)
		if a.Reference != row.reference || !slices.Equal(a.References, row.references) {
			t.Errorf("%s: %q %v, want %q %v", row.name, a.Reference, a.References, row.reference, row.references)
		}
		// An engine that predates the list reads the single reference, in a form every counted node reads (ADR 0624 §3).
		if form, _ := fs.FormOf("bundle", a.Reference); form != row.kept[0] {
			t.Errorf("%s: the single reference is in %q, want the oldest kept %q", row.name, form, row.kept[0])
		}
	}
	// An answered reference no kept form names still names where the artifact is (ADR 0498 clause 1).
	if a := fs.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{oldRef}, map[string][]string{"a": both},
		map[string][]string{"a": both}); a.Reference != oldRef {
		t.Errorf("a lone answered reference was dropped: %+v", a)
	}
	withBoth := []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: digest, Reference: oldRef, References: []string{oldRef, newRef}}}
	if got := fs.Needs(withBoth); !slices.Equal(got, []string{"form:new", "form:old"}) {
		t.Errorf("needs %v", got)
	}
	// The builder's own forms (forms.Current): a bundle a build step answers in the registry seat's form (mesh-build-go's
	// oci-artifact-registry/<module>/<artifact>@<digest>) keeps the artifact-store form it derives for each counted node
	// whose engine does not state form:seat-reference, that one in the single reference (ADR 0624 §1, §3).
	seatRef, storeRef := "oci-artifact-registry/m/code@"+digest, "artifact-store://m/code/blobs/"+digest
	seat := []string{forms.FormNeed(forms.FormSeatReference), "word:module"}
	for _, row := range []struct {
		name             string
		present, counted map[string][]string
		reference        string
		references       []string
		needs            []string
	}{
		{"seat form answered, B away stating nothing: the derived store form stays, in the single reference",
			map[string][]string{"a": seat}, map[string][]string{"a": seat, "b": nil}, storeRef, []string{storeRef, seatRef},
			[]string{"form:artifact-store", "form:seat-reference"}},
		{"seat form answered, every counted engine reads it: the seat form only",
			map[string][]string{"a": seat}, map[string][]string{"a": seat, "b": seat}, seatRef, nil, []string{"form:seat-reference"}},
		{"seat form answered, a present engine stating nothing: the store form only",
			map[string][]string{"a": seat, "b": nil}, map[string][]string{"a": seat, "b": nil}, storeRef, nil, []string{"form:artifact-store"}},
		{"seat form answered, no node present: the store form", nil, map[string][]string{"a": seat}, storeRef, nil, []string{"form:artifact-store"}},
	} {
		a := forms.Current.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{seatRef}, row.present, row.counted)
		if a.Reference != row.reference || !slices.Equal(a.References, row.references) {
			t.Errorf("%s: %q %v, want %q %v", row.name, a.Reference, a.References, row.reference, row.references)
		}
		needs := forms.Current.Needs([]delivery.BuiltArtifact{a})
		if !slices.Equal(needs, row.needs) {
			t.Errorf("%s: needs %v, want %v", row.name, needs, row.needs)
		}
		for node, stated := range row.counted {
			if forms.Current.Held(needs, stated) {
				t.Errorf("%s: the want of %s is held", row.name, node)
			}
		}
		if _, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.TrimPrefix(digest, "sha256:"), Artifacts: []delivery.BuiltArtifact{a}, Needs: needs}); err != nil {
			t.Errorf("%s: %v", row.name, err)
		}
	}
	// What the assigner holds a want by (ADR 0624 §2), from the same forms.
	for _, row := range []struct {
		name   string
		needs  []string
		stated []string
		held   bool
	}{
		{"a node stating {old}, a build carrying {old,new}: moves", []string{"form:new", "form:old"}, []string{"word:module"}, false},
		{"a node stating nothing, a build carrying the baseline's form: moves", []string{"form:old"}, nil, false},
		{"a build carrying {new} only, the node stating {old}: held", []string{"form:new"}, []string{"word:module"}, true},
		{"a build naming no form: moves", nil, nil, false},
	} {
		if got := fs.Held(row.needs, row.stated); got != row.held {
			t.Errorf("%s: held %v", row.name, got)
		}
	}
}