mesh-delivery/modules/mesh-delivery-builder/internal/build/run.go

package build

import (
	"cmp"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"maps"
	"slices"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
	sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
	sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
	repotrunk "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
	"git.novox.be/novox/mesh-sdk/go/busretry"
	"git.novox.be/novox/mesh-sdk/go/catalogue"

	"mesh-delivery-builder/internal/versions"
)

// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
type ToolchainEntries interface {
	Entries(language string) []sdkstep.NodeToolchain
}

// Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is
// handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an
// artifact to build claims the fingerprint's build key, builds it by asking the build seats (one ask per build module,
// pulled by whichever build node has a free slot: ADR 0525), and writes the key done or failed. It holds no timer; a build starts from a modules value or a
// change of the toolchains.
type Builder struct {
	// OwnModule is the module the builder runs as, its credential's: the asker of every step ask.
	OwnModule string
	Claimer   Claimer
	// StepEntry reads the entry of a fingerprint through the seat mesh-catalog's entry (design 47 G A-84, A-97).
	StepEntry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
	// Seats finds the build modules: the holders of the build seats on the seat list and the catalog half (design 47 G A-82).
	Seats *buildrun.BuildSeats
	// Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq
	// ADR 0525 §1, §3, §6; issue 1887 A4): the builder chooses no node, whichever build node pulls the ask builds it.
	Asks interface {
		Put(ctx context.Context, a sdkstep.StepAsk) error
		Wait(ctx context.Context, a sdkstep.StepAsk) (sdkstep.Step, error)
	}
	// Held says whether an ask a key names is on its stream or a build node wrote a step for it (issue 1910): a key taken
	// over at start whose ask is neither was left by a builder that stopped between naming the ask and putting it, and
	// the ask is put then, under the same id. Nil takes every named ask as held.
	Held interface {
		Held(ctx context.Context, a sdkstep.StepAsk) (bool, error)
	}
	Toolchains ToolchainEntries
	// Waits keeps the toolchain waits of design 47 G §672. Nil holds none: an entry whose fingerprint names another toolchain is
	// left unbuilt, as before.
	Waits     Waits
	Manifests Manifests
	// Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966,
	// novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the
	// builder's next start. Ended is told each key whose end was written.
	Left  func(ctx context.Context, key, why string)
	Ended func(ctx context.Context, key string)
	// Limit is how many builds the builder has asked at once; their asks wait in the build seats' streams, which are the
	// backlog (ADR 0525 §9), not here. 256 when 0.
	Limit int
	Logf  func(format string, args ...any)
	// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
	// rests on it. Nil tells nobody.
	Answered func(buildModule string)
	// Stored answers which references the artifact store no longer holds (the seat mesh-artifact-sweep's verb held; issue 1937): a done
	// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
	Stored func(ctx context.Context, references []string) (gone []string, err error)
	// Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G
	// §373, §465): the part a resolved package names in a build's dependencies. Nil names none.
	Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
	// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
	// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
	// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
	Gone interface {
		Gone(ctx context.Context, a sdkstep.StepAsk) (string, error)
	}
	CheckAfter time.Duration
	// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
	// with why; DefaultAskings when 0.
	Askings int
	// DeclineBound and DeclinePause bound and pace the asks build nodes off the agreed toolchain declined (ADR 0584 clause 2):
	// they are no askings. DefaultDeclineBound and DefaultDeclinePause when 0; a negative DeclinePause puts it again at once.
	DeclineBound, DeclinePause time.Duration
	// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
	Said func(AskedAgain)
	// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
	Log func(builds.BuildLogEntry)
	// Now is the builder's clock: of the log, and of the decline bound (DeclineBound runs from the first decline); nil is
	// time.Now. Node is the node the builder runs on, which asked the build.
	Now  func() time.Time
	Node string
	// Numbers numbers each trunk build at its claim and records what it was built with (design 51 *The versions*); nil
	// numbers nothing.
	Numbers Numbers
	// Reserved reserves the version of a trunk build at its start (design 47 G §369); nil reserves nothing.
	Reserved Reservations
	// Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine
	// answered when last asked, and counted, every node the assignments names, present or away, a node with no statement
	// stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in
	// its single reference, and its needs. Nil: no node states anything, and the first forms are kept.
	Forms func() (present, counted map[string][]string)

	once sync.Once
	sem  chan struct{}
	wg   sync.WaitGroup
	mu   sync.Mutex
	// fillWaits are the builds that said wait-held for the fill (module.fingerprint), so it is said once.
	fillWaits sync.Map
	known     map[string]meshcatalog.ModulesByModuleValue   // by modules key: the first watched branch's
	heard     map[string]meshcatalog.ModulesByModuleValue   // by branch and module (take): what supersedes what
	origin    map[string]Origin                             // by build key: the catalog revision its entry came from (take)
	edges     map[string][]meshcatalog.Edge                 // by build key: the edges of its entry (take), whose parts a package-published names
	deps      map[string]map[string][]meshbuilds.Dependency // by build key: what each artifact resolved and was built with (keepDependencies)
	inflight  map[string]bool                               // by build key
	again     map[string]meshcatalog.ModulesByModuleValue   // by build key: the newest value that arrived while the key was being built
	plans     map[string]planned                            // by manifest digest
	stepWaits map[string][]chan builds.BuildLogEntry        // by build key: the step asks waiting for its end (step.go)
	// outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them.
	outOfStep map[string][]string
	trunkFP   map[string]bool // by build key: a trunk module value carries this fingerprint
	tried     map[string]bool // by build key: looked at since this process started
	// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
	// waiting for one is built once it is here (ADR 0500 clause 6).
	published map[string]bool
	// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
	// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
	held map[string]bool
	// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
	waiting map[string]map[string]chan string
	// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
	hands map[string]*handWatch
	// events are the keys an event asks the builder to build again (Retake), until their look claims them.
	events map[string]bool
	// released are the keys builder_release is building now, whatever toolchain their fingerprint names (§276, §672).
	released map[string]bool
	// builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of
	// the module takes its older one's build out, and the asker drops that build's asks.
	builds map[string]context.CancelCauseFunc
}

type planned struct {
	plan Plan
	err  error
}

// Looking says a look of this builder holds key now, by itself or asked by hand.
func (b *Builder) Looking(key string) bool {
	b.init()
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.inflight[key] || b.hands[key] != nil
}

func (b *Builder) init() {
	b.once.Do(func() {
		limit := b.Limit
		if limit <= 0 {
			limit = 256
		}
		b.sem = make(chan struct{}, limit)
		b.trunkFP = map[string]bool{}
		b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{}
		b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{}
		b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
		b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
		b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
		b.builds = map[string]context.CancelCauseFunc{}
	})
}

func (b *Builder) logf(format string, args ...any) {
	if b.Logf != nil {
		b.Logf(format, args...)
	}
}

// Wait returns when every build started so far has ended.
func (b *Builder) Wait() { b.wg.Wait() }

// Module takes one value of a modules bucket in (the key is the module's name) and builds it if it is to be built. A value that
// is not a module is said and left. The builder takes its modules from the catalog half (Entry); this is the same take of one
// module's value, with no branch, the first one's.
func (b *Builder) Module(ctx context.Context, key string, value []byte) {
	b.init()
	m, err := meshcatalogclient.DecodeModule(value)
	if err != nil {
		b.logf("modules.%s holds what is not a module, left: %v", key, err)
		return
	}
	b.take(ctx, Origin{First: true}, key, m, nil)
}

// Origin is the catalog revision an entry came from (design 47 G §371, A-57): the watched branch it is of, as named, the source
// revision the revision was taken in from, and whether the branch is the first of the module's registration, the only one whose
// builds reserve a version and whose entry is the module's value for what the builder reads by name (a version, a build again).
type Origin struct {
	Branch string
	Source int64
	First  bool
}

// Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its
// fingerprint if it is to be built. An entry of a branch other than the first builds its fingerprint, reserves none, and
// supersedes only that branch's older one. An entry naming no module is said and left.
func (b *Builder) Entry(ctx context.Context, o Origin, e meshcatalog.CatalogEntry) {
	b.init()
	if e.Module == "" {
		b.logf("the catalog half of the branch %q holds an entry that names no module, left", o.Branch)
		return
	}
	b.take(ctx, o, e.Module, ModuleOf(e), e.Edges)
}

// EntryGone takes a module's entry out of a branch's half (the module was dropped from it, or the branch is followed no more).
func (b *Builder) EntryGone(branch string, first bool, module string) {
	b.init()
	b.mu.Lock()
	delete(b.heard, branch+"\x00"+module)
	b.mu.Unlock()
	if first {
		b.Forget(module)
	}
}

func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue, e []meshcatalog.Edge) {
	b.mu.Lock()
	hk := o.Branch + "\x00" + key
	old, had := b.heard[hk]
	b.heard[hk] = m
	if o.First {
		b.known[key] = m
	}
	var superseded context.CancelCauseFunc
	if had && old.Fp2 != "" && old.Fp2 != m.Fp2 {
		if ok, err := builds.BuilderBuildKey(key, old.Fp2); err == nil {
			superseded = b.builds[ok]
		}
	}
	if bk, err := builds.BuilderBuildKey(key, m.Fp2); err == nil && m.Fp2 != "" {
		// A fingerprint a watched branch carries is built from that branch's commit, whoever asked for it first.
		b.trunkFP[bk] = true
		b.origin[bk] = o
		b.edges[bk] = e
	}
	b.mu.Unlock()
	if superseded != nil {
		// A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn
		// from the build seats (ADR 0664 §4).
		superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2})
	}
	b.consider(ctx, key, m)
}

// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
// directory, commit, the source revision it was taken in at), the toolchains its fingerprint was computed with, the bases it is
// built against, its packages and needs, and the named paths of its artifacts.
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
	m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
		Manifest: e.ManifestDigest, Packages: e.Packages, Needs: e.Needs}
	if len(e.Bases) > 0 {
		m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases}
	}
	if in := e.Inputs; in != nil {
		m.Src, m.Toolchains, m.Problems = in.Src, in.Toolchains, in.Problems
		if len(in.Paths) > 0 {
			if m.Inputs2 == nil {
				m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{}
			}
			m.Inputs2.Paths = map[string]string{}
			for artifact, paths := range in.Paths {
				for _, p := range paths {
					m.Inputs2.Paths[p] = artifact
				}
			}
		}
	}
	return m
}

// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build
// of the trunk's value was before branches were watched.
func (b *Builder) originOf(key string) Origin {
	b.mu.Lock()
	defer b.mu.Unlock()
	if o, ok := b.origin[key]; ok {
		return o
	}
	return Origin{First: true}
}

// NodesChanged is Reconsider after a toolchain moved, so a build that waited for the nodes to agree on one may go on.
func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) }

// Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key
// waiting for one is then looked at again, ADR 0500 clause 6), and closes every key left building for a fingerprint its
// module no longer has (issue 1660, ADR 0478 clause 6): the key can never be built, since nothing asks for it, and would
// stay building for ever. It fails the key naming who left it. A key of the fingerprint the module has now is not
// touched: the build takes it over. Call it once the modules bucket has been read, so every module is known.
//
// pictured are the build keys the followed halves' pictures name as current: a follower is ready before it has told its
// entries (views.Follower closes Ready, then calls Changed), so at ready the entries may not be taken in yet, and a key of
// a fingerprint a picture holds is never closed for want of the entry's arrival (the restart that put its asks a second time).
func (b *Builder) Sweep(ctx context.Context, pictured ...string) error {
	b.init()
	keys, err := b.Claimer.Store.Keys(ctx)
	if err != nil {
		return err
	}
	b.mu.Lock()
	current := map[string]bool{}
	for _, k := range pictured {
		current[k] = true
	}
	// What every followed branch carries now is current, not only the first branch's (design 47 G §348).
	for _, held := range []map[string]meshcatalog.ModulesByModuleValue{b.known, b.heard} {
		for hk, m := range held {
			name := hk[strings.LastIndex(hk, "\x00")+1:]
			if m.Fp2 != "" {
				if k, err := builds.BuilderBuildKey(name, m.Fp2); err == nil {
					current[k] = true
				}
			}
		}
	}
	b.mu.Unlock()
	newly := false
	for _, key := range keys {
		value, rev, found, err := b.Claimer.Store.Get(ctx, key)
		if err != nil || !found {
			continue
		}
		cur, err := builds.DecodeBuilderBuild(value)
		if err != nil {
			continue
		}
		newly = b.notePublished(cur) || newly
		if current[key] || cur.State != builds.BuildBuilding {
			continue
		}
		closed := builds.BuilderBuild{State: builds.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step,
			Error: fmt.Sprintf("closed at start: left building by %s, and the module no longer has this fingerprint, so nothing will build it", cur.By)}
		data, err := builds.EncodeBuilderBuild(closed)
		if err != nil {
			return err
		}
		if _, err := b.Claimer.Store.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
			return fmt.Errorf("%s: %w", key, err)
		}
		b.logf("%s: closed, left building by %s for a fingerprint the module no longer has", key, cur.By)
	}
	if newly {
		// A key read before the scan may wait for a package the scan found published (ADR 0500 clause 6).
		b.Reconsider(ctx)
	}
	return nil
}

// notePublished keeps the package versions a done build names as references, and says whether one was new.
func (b *Builder) notePublished(v builds.BuilderBuild) bool {
	if v.State != builds.BuildDone {
		return false
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	newly := false
	for _, a := range v.Artifacts {
		if a.Kind == catalogue.ArtifactPackage && a.Reference != "" && !b.published[a.Reference] {
			b.published[a.Reference] = true
			newly = true
		}
	}
	return newly
}

func (b *Builder) isPublished(ref string) bool {
	b.mu.Lock()
	defer b.mu.Unlock()
	return b.published[ref]
}

// Forget takes a module out (its key was deleted).
func (b *Builder) Forget(key string) {
	b.init()
	b.mu.Lock()
	delete(b.known, key)
	b.mu.Unlock()
}

// storedOnce is Stored for one key, asked once per process while the store answers every reference held.
func (b *Builder) storedOnce(key string) func(ctx context.Context, references []string) ([]string, error) {
	if b.Stored == nil {
		return nil
	}
	return func(ctx context.Context, references []string) ([]string, error) {
		id := key + " " + strings.Join(references, " ")
		b.mu.Lock()
		known := b.held[id]
		b.mu.Unlock()
		if known {
			return nil, nil
		}
		gone, err := b.Stored(ctx, references)
		if err == nil && len(gone) == 0 {
			b.mu.Lock()
			b.held[id] = true
			b.mu.Unlock()
		}
		return gone, err
	}
}

// ArtifactsGone takes the seat mesh-artifact-sweep's event artifacts-gone in (issue 1937, ADR 0571 §2): the sweep found
// that the store does not hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
// builder: what is built rests on the store's answer, never on the event alone.
func (b *Builder) ArtifactsGone(ctx context.Context, e sweep.ArtifactsGone) {
	b.init()
	b.mu.Lock()
	m, ok := b.known[e.Module]
	if key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint); err == nil {
		for id := range b.held {
			if strings.HasPrefix(id, key+" ") {
				delete(b.held, id)
			}
		}
	}
	b.mu.Unlock()
	if !ok || m.Fp2 != e.Fingerprint {
		b.logf("%s: heard %s for %s, a fingerprint the module does not have now: nothing to build", e.Module,
			sweep.EventArtifactsGone, e.Fingerprint)
		return
	}
	b.logf("%s: heard %s (%s): asking the artifact store and building the key again", e.Module,
		sweep.EventArtifactsGone, strings.Join(e.Gone, ", "))
	b.consider(ctx, e.Module, m)
}

// Reconsider looks at every module again: the toolchains changed, so a build that failed for them may build now.
func (b *Builder) Reconsider(ctx context.Context) {
	b.init()
	b.mu.Lock()
	known := make(map[string]meshcatalog.ModulesByModuleValue, len(b.known))
	for k, m := range b.known {
		known[k] = m
	}
	b.mu.Unlock()
	for _, k := range sortedKeys(known) {
		b.consider(ctx, k, known[k])
	}
}

func sortedKeys(m map[string]meshcatalog.ModulesByModuleValue) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// planOf is the plan of the manifest with this digest, read once: a manifest by digest never changes. A manifest that
// cannot be read is not kept, so it is read again when the module is.
func (b *Builder) planOf(ctx context.Context, digest string) (Plan, error, bool) {
	b.mu.Lock()
	p, ok := b.plans[digest]
	b.mu.Unlock()
	if ok {
		return p.plan, p.err, true
	}
	m, err := b.Manifests.Manifest(ctx, digest)
	if err != nil {
		return Plan{}, fmt.Errorf("its manifest %s cannot be read: %w", digest, err), false
	}
	plan, err := PlanOf(m, b.Seats)
	var none *NoBuildModule
	if errors.As(err, &none) {
		return plan, err, false // the seat list or the catalog half names no build module yet: made again when they do
	}
	b.mu.Lock()
	b.plans[digest] = planned{plan, err}
	b.mu.Unlock()
	return plan, err, true
}

func (b *Builder) consider(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) {
	if m.Fp2 == "" || m.Manifest == "" {
		return
	}
	key, err := builds.BuilderBuildKey(module, m.Fp2)
	if err != nil {
		b.logf("modules.%s: %v", module, err)
		return
	}
	b.mu.Lock()
	if b.inflight[key] {
		b.again[key] = m // looked at once more when the build ends, so a change is never lost
		b.mu.Unlock()
		return
	}
	b.inflight[key] = true
	b.mu.Unlock()
	b.wg.Add(1)
	go func() {
		defer b.wg.Done()
		b.sem <- struct{}{}
		defer func() { <-b.sem }()
		for {
			b.try(ctx, module, m)
			b.mu.Lock()
			next, more := b.again[key]
			delete(b.again, key)
			if !more || ctx.Err() != nil {
				delete(b.inflight, key)
				b.mu.Unlock()
				return
			}
			b.mu.Unlock()
			m = next
		}
	}()
}

// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
func (b *Builder) startOnce(ctx context.Context, key, module string, m meshcatalog.ModulesByModuleValue) {
	b.wg.Add(1)
	go func() {
		defer b.wg.Done()
		b.sem <- struct{}{}
		defer func() { <-b.sem }()
		for {
			b.try(ctx, module, m)
			b.mu.Lock()
			next, more := b.again[key]
			delete(b.again, key)
			if !more || ctx.Err() != nil {
				delete(b.inflight, key)
				b.mu.Unlock()
				return
			}
			b.mu.Unlock()
			m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
		}
	}()
}

// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) {
	key, _ := builds.BuilderBuildKey(module, m.Fp2)
	defer b.handEnded(key)
	defer b.unrelease(key)
	plan, planErr, read := b.planOf(ctx, m.Manifest)
	var noModule *NoBuildModule
	if errors.As(planErr, &noModule) {
		// The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again
		// when they do (Reconsider, at each change of either; design 47 G A-82).
		b.logf("%s at %s is not built yet: %v", module, m.Fp2, planErr)
		return
	}
	nothing := read && planErr == nil && len(plan.Tasks) == 0
	if planErr == nil && !nothing && !b.isReleased(key) {
		if why := b.stale(m, plan); why != "" {
			if w, ok := b.toolchainWait(module, m, plan); ok && b.Waits != nil {
				b.Waits.Hold(ctx, w)
			}
			// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
			// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
			b.logf("%s at %s is not built: %s", module, m.Fp2, why)
			return
		}
		if b.Waits != nil {
			b.Waits.End(ctx, module, m.Fp2, WaitRan) // a toolchain-agreed names the version the fingerprint names
		}
	}
	b.mu.Lock()
	trunk := b.trunkFP[key]
	first := !b.tried[key]
	b.tried[key] = true
	byHand := b.hands[key] != nil
	byEvent := b.events[key]
	delete(b.events, key)
	b.mu.Unlock()
	var numbering versions.Numbering
	if trunk && planErr == nil && !nothing {
		var goOn bool
		if numbering, goOn = b.number(ctx, module, m, plan); !goOn {
			return
		}
	}
	// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
	reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
	if reserving && b.waitFill(module, m) {
		return
	}
	// A key built again by hand is taken over as it stands too: one left building by a look that ended without its write
	// (issue 1966) is the hand tool's to finish.
	cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished,
		Stored: b.storedOnce(key)})
	if err != nil {
		b.logf("%s: %v", module, err)
		return
	}
	if cl == nil {
		return
	}
	b.handClaimed(key, cl.Rev)
	if b.Waits != nil {
		b.Waits.End(ctx, module, m.Fp2, WaitPublished) // a key queued behind a package is claimed once it is published
	}
	b.logBuild(LogStarted(module, m, "", b.Node, b.now()))
	if cl.Unasked != nil {
		b.logf("%v", cl.Unasked)
	}
	if len(cl.Gone) > 0 {
		// Said aloud (issue 1937): the key was done, and the store's collector deleted what it names.
		b.logf("%s: built again: the artifact store no longer holds %s", cl.Key, strings.Join(cl.Gone, ", "))
	}
	// A done key taken again builds only what is missing: the image built unpublished as a candidate's (A-289); the
	// other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6).
	var prior *builds.BuilderBuild
	var only map[string]bool
	publishing := false
	if cl.Build.State == builds.BuildDone {
		prior, only = &cl.Build, map[string]bool{}
		if trunk {
			for _, bm := range publishable(cl.Build, b.Seats, imageSeat) {
				only[bm] = true
				publishing = true
			}
		}
	}
	var reserved versions.ReservedVersion
	hasReserved := false
	if reserving && cl.Build.State != builds.BuildDone {
		reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key))
	}
	var result builds.BuilderBuild
	if numbering.Refused != "" && cl.Build.State != builds.BuildDone {
		// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
		// fails with why, and is taken again at the next look.
		result = failed("%s", numbering.Refused)
	} else if nothing {
		// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
		// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
		result = builds.BuilderBuild{State: builds.BuildDone, Step: "nothing to build: the manifest names no artifact"}
	} else {
		run, withdraw := context.WithCancelCause(ctx)
		b.mu.Lock()
		b.builds[key] = withdraw
		b.mu.Unlock()
		result = b.build(run, module, m, plan, planErr, prior, only, b.notes(cl), reserved.Version)
		b.mu.Lock()
		delete(b.builds, key)
		b.mu.Unlock()
		withdraw(nil)
		var w buildrun.Withdrawn
		if ctx.Err() == nil && errors.As(context.Cause(run), &w) {
			// Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them).
			result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})}
		}
	}
	if ctx.Err() != nil {
		return
	}
	if result.State == "" {
		b.logf("%s: left building, the toolchains moved under it", cl.Key)
		return
	}
	result.By, result.Manifest, result.EventBuilds = b.Claimer.By, m.Manifest, cl.Build.EventBuilds
	if len(cl.Gone) > 0 && result.State == builds.BuildDone {
		result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
	}
	if publishing && result.State == builds.BuildFailed {
		// The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the
		// checker and the failed-key retake see it, and a look does not build the image again for nothing.
		result.Error = keyError([]string{"publishing what the merged commit built failed: " + result.Error})
	}
	if err := b.Claimer.Finish(ctx, cl, result); err != nil {
		b.logf("%s: the build could not be written: %v", cl.Key, err)
		if b.Left != nil && !errors.Is(err, ErrConflict) && ctx.Err() == nil {
			b.Left(ctx, cl.Key, err.Error()) // left building with no look holding it (issue 1966)
		}
		return
	}
	if b.Ended != nil {
		b.Ended(ctx, cl.Key)
	}
	ended := LogEnded(module, m, "", b.Node, b.now(), result)
	ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
	b.logBuild(ended)
	if result.State == builds.BuildDone && b.Numbers != nil && len(numbering.Versions) > 0 {
		if err := b.Numbers.Built(ctx, module, m, plan, numbering, result.Artifacts); err != nil {
			b.logf("%s: %v", cl.Key, err)
		}
	}
	if result.State == builds.BuildDone && hasReserved {
		digests := map[string]string{}
		for _, a := range result.Artifacts {
			digests[module+":"+a.Name] = a.Digest
		}
		if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
			b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
		}
	}
	if result.State == builds.BuildFailed {
		b.logf("%s: failed: %s", cl.Key, result.Error)
	}
	if result.State == builds.BuildQueued {
		b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits)
		if b.Waits != nil {
			// Said as a wait in the one wait form, with its part, for the watchman to time (design 47 G A-192; ADR 0603 §8).
			b.Waits.Hold(ctx, ToolchainWait{Module: module, Fingerprint: m.Fp2, Package: result.Waits, Since: b.now()})
		}
	}
	if b.notePublished(result) {
		// A key may wait for the package this build published (ADR 0500 clause 6).
		b.Reconsider(ctx)
	}
}

// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
func (b *Builder) stale(m meshcatalog.ModulesByModuleValue, plan Plan) string {
	for _, bm := range plan.Modules() {
		if !plan.NeedsToolchain(bm) {
			continue // an archive is packed with no toolchain
		}
		language, _ := b.Seats.LanguageOf(bm)
		ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language))
		if ag.Wait != "" {
			return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
		}
		tc := ag.Toolchain
		want, ok := m.Toolchains[language]
		if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
			return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
		}
	}
	return ""
}

func failed(format string, args ...any) builds.BuilderBuild {
	return builds.BuilderBuild{State: builds.BuildFailed, Error: fmt.Sprintf(format, args...)}
}

// lane is the build of a plan's tasks of one build module on one node.
type lane struct {
	module, node string // node is the one that pulled the lane's ask
	version      string // the version the build reserved, MAJOR.MINOR.PATCH, empty for a build that reserves none (design 47 G §573)
	tasks        []Task
	toolchain    *builds.ToolchainUsed
	digests      map[string]string                // artifact -> digest
	references   map[string]string                // artifact -> the reference an image was pushed to (a trunk build only)
	imports      []string                         // the mesh packages the lane's programs import, as its build module answered them
	resolved     map[string][]sdkstep.AskResolved // artifact -> the mesh packages its build resolved (design 47 G §466)
	tcDigest     string                           // the digest of the toolchain program the building node ran, kind 3's (design 47 G §466, §472)
	tcVersion    string                           // and the version it ran
	kept         string                           // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty
	outOfStep    []string                         // the generated files its build module's compare found out of step, passed or failed
	err          error
}

func (b *Builder) build(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, planErr error, prior *builds.BuilderBuild, only map[string]bool, asked *askNotes, version string) builds.BuilderBuild {
	switch {
	case planErr != nil:
		return failed("%s", planErr)
	case m.Path == "":
		// Empty is not known (a root module's path is "."): the indexer writes it again (issue 1615).
		return failed("the module's key names no directory (path), so the build tools cannot be told where %s lives", module)
	}
	repository := repotrunk.RepositoryName(m.Repository)
	var lanes []*lane
	used := map[string]builds.ToolchainUsed{}
	var sitting []string // the nodes that sit out of a language's build, said in the key's step (ADR 0584 clause 2)
	var kept []string    // build modules whose artifacts stay as the prior done key has them
	for _, bm := range plan.Modules() {
		if prior != nil && len(only) > 0 && !only[bm] {
			kept = append(kept, bm)
			continue
		}
		var toolchain *builds.ToolchainUsed
		if plan.NeedsToolchain(bm) {
			// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
			language, _ := b.Seats.LanguageOf(bm)
			ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language))
			if ag.Wait != "" {
				return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
			}
			tc := ag.Toolchain
			sitting = append(sitting, SittingOutWords(language, ag)...)
			released := false
			if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
				released = b.isReleased(key) // builder_release builds it with the running build module (§276)
			}
			if want := m.Toolchains[language]; !released && (want.Version != tc.Version || want.Recipe != tc.Recipe) {
				return builds.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
			}
			u := builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
			used[language], toolchain = u, &u
		}
		var tasks []Task
		for _, t := range plan.Tasks {
			if t.Module == bm {
				tasks = append(tasks, t)
			}
		}
		// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
		lanes = append(lanes, &lane{module: bm, version: version, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}, resolved: map[string][]sdkstep.AskResolved{}})
	}
	if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
		b.handLanes(key, len(lanes))
	}
	var wg sync.WaitGroup
	for _, l := range lanes {
		wg.Add(1)
		go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module, asked) }()
	}
	wg.Wait()
	if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
		b.keepOutOfStep(key, lanes)
		b.keepDependencies(key, module, lanes)
	}
	var errs []string
	for _, l := range lanes {
		if l.err != nil {
			errs = append(errs, l.err.Error())
		}
	}
	if len(errs) > 0 {
		if ref := waitsOf(errs); ref != "" {
			// The module's lock names a version of a package the registry does not hold yet: the key waits for it, queued, and
			// is not failed (ADR 0500 clause 6).
			return builds.BuilderBuild{State: builds.BuildQueued, Waits: ref, Step: "waits for the package " + ref + " to be published"}
		}
		return failed("%s", keyError(errs))
	}
	var artifacts []builds.BuiltArtifact
	var present, counted map[string][]string
	if b.Forms != nil {
		present, counted = b.Forms()
	}
	keptModule := map[string]bool{}
	for _, bm := range kept {
		keptModule[bm] = true
	}
	for _, t := range plan.Tasks {
		if keptModule[t.Module] {
			found := false
			for _, a := range prior.Artifacts {
				if a.Name == t.Artifact {
					artifacts, found = append(artifacts, a), true
				}
			}
			if !found {
				return failed("the done build holds no artifact %s to keep", t.Artifact)
			}
			continue
		}
		byNode := map[string]string{}
		for _, l := range lanes {
			if d, ok := l.digests[t.Artifact]; ok && l.module == t.Module {
				byNode[l.node] = d
			}
		}
		d, err := buildrun.Agree(t.Artifact, byNode)
		if err != nil {
			return failed("%s", err)
		}
		built := builds.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d}
		var answered []string
		for _, l := range lanes {
			if l.module != t.Module {
				continue
			}
			r := l.references[t.Artifact]
			if r == "" && stored(t.Kind) && (l.kept == "" || strings.TrimPrefix(d, "sha256:") != l.kept) {
				// A bundle, an archive or an upstream copy is done only when its push was answered with where it is (ADR 0498
				// clause 1); a candidate's included (clause 3).
				return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", builds.NotStored, t.Artifact, l.node)
			}
			if r != "" {
				answered = append(answered, r)
			}
		}
		// The forms a counted node's engine reads are kept, the oldest in the single reference (ADR 0624 §1, §3).
		artifacts = append(artifacts, forms.Current.Choose(built, answered, present, counted))
	}
	var steps []string
	for _, l := range lanes {
		steps = append(steps, fmt.Sprintf("%s on %s", l.module, l.node))
	}
	for _, bm := range kept {
		steps = append(steps, bm+" kept")
		kept, _ := b.Seats.LanguageOf(bm)
		for lang, u := range prior.Toolchains {
			if _, ok := used[lang]; !ok && kept == lang {
				used[lang] = u
			}
		}
	}
	step := "built " + strings.Join(steps, "; ")
	if len(sitting) > 0 {
		step += " (" + strings.Join(sitting, "; ") + ")"
	}
	if len(used) == 0 {
		used = nil
	}
	done := builds.BuilderBuild{State: builds.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used,
		Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1)
	for _, l := range lanes {
		if l.kept != "" {
			done.KeptOn = l.node // design 47 G §671: the build node that keeps the build
		}
	}
	if _, err := builds.EncodeBuilderBuild(done); err != nil {
		// A reference the build module answered that does not name the artifact and its digest is no reference.
		return failed("%s%v", builds.NotStored, err)
	}
	return done
}

// stored is whether an artifact of this kind is in the artifact store once built, a candidate's too, so its done key
// names its reference (ADR 0498 clauses 1, 3 and 4); an image or a package names one only when built for the trunk.
func stored(kind string) bool {
	return kind == catalogue.ArtifactBundle || kind == catalogue.ArtifactArchive || kind == catalogue.ArtifactUpstream
}

// waitsFor opens the reason of a fetch step whose module's lock names a package version the registry does not hold, then
// `<name>@<version>` (the Node build module's, ADR 0500 clause 6).
const waitsFor = builds.WaitsFor

// waitsOf is the package version a lane's error says the module waits for, or "" when none does or what it names is not
// `<name>@<version>`.
func waitsOf(errs []string) string {
	for _, e := range errs {
		at := strings.Index(e, waitsFor)
		if at < 0 {
			continue
		}
		ref, _, _ := strings.Cut(e[at+len(waitsFor):], " ")
		ref = strings.TrimRight(ref, ":;,")
		if _, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildQueued, Waits: ref}); err == nil {
			return ref
		}
	}
	return ""
}

// keyPrefixes are the errors a build module's step may open its reason with that the key's error must open with too, so
// the claim reads them (builds.FailedUntilChanged), first the ones not built again.
var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.RefusedKind, builds.NotStored}

// keyError is the error of a key from its lanes' errors: when one names a prefix of keyPrefixes inside the step's words
// (`<tool> on <node>: the step <key> failed: not stored: ...`), the key's error opens with that prefix, so a reader of the
// key need not search for it.
func keyError(errs []string) string {
	// A transport fault is the key's own words only when every lane failed on one: beside a real failure it is said after it,
	// so the key is not built again on an event for a failure the source made (design 47 G §568).
	errs = slices.Clone(errs)
	slices.SortStableFunc(errs, func(a, b string) int {
		return boolInt(strings.HasPrefix(a, builds.TransportFaultPrefix)) - boolInt(strings.HasPrefix(b, builds.TransportFaultPrefix))
	})
	for _, prefix := range keyPrefixes {
		for i, e := range errs {
			if strings.HasPrefix(e, prefix) {
				return strings.Join(errs, "; ")
			}
			if at := strings.Index(e, prefix); at >= 0 {
				rest := append([]string{prefix + e[:at] + e[at+len(prefix):]}, append(append([]string(nil), errs[:i]...), errs[i+1:]...)...)
				return strings.Join(rest, "; ")
			}
		}
	}
	return strings.Join(errs, "; ")
}

// imageSeat says an image build is asked of the image build seat, which takes it when a node pulls it: a key failed for
// want of an image builder before R4g is taken again (ADR 0525 §1).
func imageSeat() bool { return true }

// askNotes writes a claimed key's asks into it, building, before each is put (ADR 0525 §1; issue 1887 A4b), and holds
// the asks a key taken over at start already names, so they are waited for and never put again.
type askNotes struct {
	mu      sync.Mutex
	b       *Builder
	cl      *Claim
	resumed map[string]string
	// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
	step string
}

func (b *Builder) notes(cl *Claim) *askNotes {
	n := &askNotes{b: b, cl: cl}
	if cl.Build.State == builds.BuildBuilding && len(cl.Build.Asks) > 0 {
		n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
	}
	return n
}

// resume is the id of the ask a taken-over key names for language, or "".
func (n *askNotes) resume(language string) string {
	n.mu.Lock()
	defer n.mu.Unlock()
	return n.resumed[language]
}

// note writes the ask's id into the key, building, by compare-and-set at the claim's revision.
func (n *askNotes) note(ctx context.Context, language, id, step string) error {
	n.mu.Lock()
	defer n.mu.Unlock()
	v := n.cl.Build
	v.State, v.By, v.Step, v.Error, v.Artifacts, v.Needs = builds.BuildBuilding, n.b.Claimer.By, step, "", nil, nil
	v.Asks = maps.Clone(v.Asks)
	if v.Asks == nil {
		v.Asks = map[string]string{}
	}
	v.Asks[language] = id
	return n.b.Claimer.Finish(ctx, n.cl, v)
}

// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
// check stream for a candidate's) and reads what the node that pulled it built (ADR 0525 §1, §3, §6). An upstream artifact
// is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches nothing. A lane of a
// key taken over at start waits for the ask the key names and puts none.
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) {
	req := sdkstep.Fetch{Repository: repository, Commit: m.Commit}
	if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
		// A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from
		// the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e).
		req.Dir = m.Path
		for p := range m.Inputs2.Paths {
			req.Paths = append(req.Paths, p)
		}
		sort.Strings(req.Paths)
	}
	seat, _ := b.Seats.OfModule(l.module)
	language := seat.AskLanguage()
	kind := sdkstep.AskDelivery
	job := &sdkstep.BuildJob{Module: module, Path: m.Path, Fp: m.Fp2, Toolchain: buildToolchain(language, l.toolchain),
		Publish: true}
	for _, t := range l.tasks {
		a := sdkstep.AskArtifact{Artifact: t.Artifact}
		if t.Kind == catalogue.ArtifactUpstream {
			a.From = t.From // the copy runs no code of a change: the module's name, the artifact and its pin
		} else if seat, _, ok := versions.SeatOf(t.Kind, t.Language); ok && l.version != "" {
			// A build under a reserved version names the registry seat each artifact is published through, so a Go package is
			// published and tagged under it (design 47 G §370; design 51 *Publishing*).
			a.Seat = seat
		}
		job.Artifacts = append(job.Artifacts, a)
	}
	if needsFetch(l.tasks) {
		job.Fetch = &req
	}
	ask := sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: kind, Language: language, Asker: b.OwnModule, At: time.Now(), Version: l.version, Build: job}
	if language == sdkstep.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
		// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
		// them into the workspace, whatever the repository's go.mod names.
		ask.Packages = maps.Clone(m.Packages)
		for _, fb := range req.Builds {
			delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
		}
	}
	if id := asked.resume(language); id != "" {
		ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
		if b.Held != nil {
			held, err := b.Held.Held(ctx, ask)
			if err != nil {
				// Said, and the key is not failed: the ask may well be on its stream and its build finish (issue 1910).
				b.logf("whether the ask %s of %s the key names is on its stream cannot be read, so it is waited for as put: %v", ask.ID, l.module, err)
				held = true
			}
			if !held {
				// The builder that named it stopped before it put it (issue 1910): put it now, under the same id.
				b.logf("the ask %s of %s that the key names was never put: it is put now", ask.ID, l.module)
				if err := b.Asks.Put(ctx, ask); err != nil {
					l.err = err
					return
				}
			}
		}
	} else {
		if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil {
			// The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from
			// the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3).
			l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err)
			if busretry.TimedOut(err) || busretry.Undelivered(err) {
				seat, _ := sdkstep.BuildSeat(language)
				l.err = errors.New(buildrun.TransportFault(buildrun.FaultClass(err.Error()), seat, "no node", l.err.Error()))
			}
			return
		}
		if err := b.Asks.Put(ctx, ask); err != nil {
			l.err = err
			return
		}
		// A request to build the key again by hand is answered the ask as asked once it is put, not once it is noted: a
		// put the stream refuses after the note answers the key's failure (BuildAgainResult.Asks).
		b.handAsk(asked.cl.Key, language, ask.ID)
	}
	step, ask, err := b.answer(ctx, l, ask, asked, module, m.Fp2)
	if err != nil {
		l.err = fmt.Errorf("%s: %w", l.module, err)
		return
	}
	l.node = step.Node
	// The build module's compare of a contracts module's generated files (design 26, CAT-BG5/BN2) answers the files out
	// of step as sdkstep.BuildAnswer.OutOfStep, in a failed step's result too: read before the step's state is.
	var compared sdkstep.BuildAnswer
	if len(step.Result) > 0 && json.Unmarshal(step.Result, &compared) == nil {
		l.outOfStep = compared.OutOfStep
	}
	if step.State != sdkstep.StepPassed {
		l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason)
		if strings.HasPrefix(step.Reason, builds.TransportFaultPrefix) {
			l.err = errors.New(step.Reason) // a key's error opens with it (design 47 G §568)
		}
		return
	}
	var answer sdkstep.BuildAnswer
	if err := json.Unmarshal(step.Result, &answer); err != nil {
		l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result)
		return
	}
	l.imports = answer.Imports
	l.kept = answer.Kept
	if answer.Toolchain != nil {
		l.tcDigest, l.tcVersion = answer.Toolchain.Digest, answer.Toolchain.Version
	}
	built := map[string]sdkstep.AskBuilt{}
	for _, a := range answer.Artifacts {
		built[a.Artifact] = a
	}
	for _, t := range l.tasks {
		a, ok := built[t.Artifact]
		if !ok || a.Digest == "" {
			l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, step.Node, step.Result)
			return
		}
		l.digests[t.Artifact] = a.Digest
		l.resolved[t.Artifact] = a.Resolved
		if a.Reference != "" {
			l.references[t.Artifact] = a.Reference
		}
		if a.Reference != "" && b.Answered != nil {
			b.Answered(l.module)
		}
	}
}

// needsFetch is whether any of the tasks builds from the module's source: every one but an upstream copy.
func needsFetch(tasks []Task) bool {
	for _, t := range tasks {
		if t.Kind != catalogue.ArtifactUpstream {
			return true
		}
	}
	return false
}

func (b *Builder) now() time.Time {
	if b.Now != nil {
		return b.Now()
	}
	return time.Now()
}

// logBuild tells the log an entry, said in the journal when it cannot be one.
func (b *Builder) logBuild(e builds.BuildLogEntry) {
	b.stepEnded(e)
	if b.Log == nil {
		return
	}
	if _, err := builds.EncodeBuildLogEntry(e); err != nil {
		b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err)
		return
	}
	b.Log(e)
}

// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
// that asked it and the time.
func LogStarted(module string, m meshcatalog.ModulesByModuleValue, ref, by string, at time.Time) builds.BuildLogEntry {
	dir := m.Path
	if dir == "" {
		dir = meshcatalogclient.RootPath
	}
	return builds.BuildLogEntry{Event: builds.BuildStartedEvent, Module: module, FP: m.Fp2, Source: strconv.FormatUint(uint64(m.Src), 10),
		Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()}
}

// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
// was built against and each artifact made, as the done key has them: what the operator's command nox builds shows of a build.
func LogEnded(module string, m meshcatalog.ModulesByModuleValue, ref, by string, at time.Time, result builds.BuilderBuild) builds.BuildLogEntry {
	e := LogStarted(module, m, ref, by, at)
	e.Event, e.Outcome, e.Manifest = builds.BuildEndedEvent, result.State, m.Manifest
	if result.Manifest != "" {
		e.Manifest = result.Manifest
	}
	if m.Inputs2 != nil {
		e.Against = append([]string(nil), m.Inputs2.Bases...)
	}
	e.Made = append([]builds.BuiltArtifact(nil), result.Artifacts...)
	e.KeptOn = result.KeptOn
	e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
	e.Error, e.Step = result.Error, result.Step          // a failed build's reason, not only its outcome
	e.Needs = append([]string(nil), result.Needs...)
	return e
}

// keepOutOfStep holds, by build key, the out-of-step files the lanes' build modules answered, sorted and each once, until
// the build's end entry takes them (takeOutOfStep); a build with none holds nothing.
func (b *Builder) keepOutOfStep(key string, lanes []*lane) {
	seen := map[string]bool{}
	for _, l := range lanes {
		for _, f := range l.outOfStep {
			seen[f] = true
		}
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	if len(seen) == 0 {
		delete(b.outOfStep, key)
		return
	}
	if b.outOfStep == nil {
		b.outOfStep = map[string][]string{}
	}
	b.outOfStep[key] = slices.Sorted(maps.Keys(seen))
}

// takeOutOfStep answers the out-of-step files held for a build key and forgets them.
func (b *Builder) takeOutOfStep(key string) []string {
	b.mu.Lock()
	defer b.mu.Unlock()
	files := b.outOfStep[key]
	delete(b.outOfStep, key)
	return files
}

// importsOf are the mesh packages the lanes' programs import, as their build modules answered them (novox/hq ADR 0593
// §2): sorted and each once; nil when none answered any. A lane kept from a prior build answered none, so a build that
// kept one reports only what it built.
func importsOf(lanes []*lane) []string {
	seen := map[string]bool{}
	for _, l := range lanes {
		for _, p := range l.imports {
			seen[p] = true
		}
	}
	if len(seen) == 0 {
		return nil
	}
	return slices.Sorted(maps.Keys(seen))
}

// SittingOutWords says, per node that is not on the agreed toolchain of a language, that it sits out and which version
// it has against the agreed one: "ace sits out: typescript 26.12.1, agreed 26.13.0" (novox/hq ADR 0584 clause 2).
func SittingOutWords(language string, a sdkstep.Agreement) []string {
	var out []string
	for _, s := range a.SittingOut {
		out = append(out, fmt.Sprintf("%s sits out: %s %s, agreed %s", s.Node, language, s.Version, a.Toolchain.Version))
	}
	return out
}

// keepDependencies keeps, for a build key, what its step results say each artifact resolved (kind 1) and was built with (kind 3) as
// the dependencies `package-published` carries (design 47 G §466, §673, A-57): the part of a kind-1 item is the artifact reference of
// the package, and of the kind-3 item the toolchain's part, each as the edges of the entry name it. A part no edge names is said and
// its item left out.
func (b *Builder) keepDependencies(key, module string, lanes []*lane) {
	b.mu.Lock()
	edges := b.edges[key]
	b.mu.Unlock()
	out := map[string][]meshbuilds.Dependency{}
	for _, l := range lanes {
		for _, t := range l.tasks {
			ref := module + ":" + t.Artifact
			var deps []meshbuilds.Dependency
			for _, r := range l.resolved[t.Artifact] {
				part := ""
				for _, e := range edges {
					if e.Kind == 1 && e.To == ref && e.Why == r.Package {
						part = cmp.Or(e.Part, e.From)
					}
				}
				if part == "" && b.Members != nil {
					if mod, art, ok, err := b.Members(context.Background(), r.Package); ok && err == nil {
						part = mod + ":" + art
					}
				}
				if part == "" {
					b.logf("%s: the package %s the build of %s resolved names no artifact: its dependency is not said", key, r.Package, ref)
					continue
				}
				deps = append(deps, meshbuilds.Dependency{Kind: 1, Part: part, Version: r.Version, Digest: r.Digest})
			}
			if l.tcDigest != "" {
				part := ""
				for _, e := range edges {
					if e.Kind == 3 && e.To == ref {
						part = cmp.Or(e.Part, e.From)
					}
				}
				if part == "" {
					b.logf("%s: no edge of kind 3 names the toolchain part the build of %s used: its dependency is not said", key, ref)
				} else {
					deps = append(deps, meshbuilds.Dependency{Kind: 3, Part: part, Version: l.tcVersion, Digest: l.tcDigest})
				}
			}
			if len(deps) > 0 {
				out[ref] = deps
			}
		}
	}
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.deps == nil {
		b.deps = map[string]map[string][]meshbuilds.Dependency{}
	}
	if len(out) == 0 {
		delete(b.deps, key)
		return
	}
	b.deps[key] = out
}

// takeDependencies answers the dependencies held for a build key and forgets them.
func (b *Builder) takeDependencies(key string) map[string][]meshbuilds.Dependency {
	b.mu.Lock()
	defer b.mu.Unlock()
	d := b.deps[key]
	delete(b.deps, key)
	return d
}