BL14: the builder holds an entry whose fingerprint names another toolchain, in its state waits, with wait-held and wait-ended

An entry whose fingerprint names a toolchain version of its language other than
the agreed one is held in the state waits (one key <module>.<fingerprint>), says
wait-held and wait-ended (ran when the agreed toolchain names the version,
released by builder_release) on its stream waits, says wait-held again for every
key at its start; builder_waits and builder_release; its setting
toolchain-wait-minutes. Not built: the bound's finding toolchain-waits (its
scope is not in the design).

Builds: design 47 G §672, §267, §276, §633
Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

@@ -372,6 +372,15 @@ func main() {
 	}}
 	tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr)
 	tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
+	ws := stdio.State(build.WaitsState)
+	wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
+		e, err := ws.Get(key)
+		if err != nil || e == nil {
+			return nil, false, err
+		}
+		return e.Value, true, nil
+	}}
+	tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
 	tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
 	// The builder's expiring keys (state bounds): what each bounds and when it ends.
 	tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
+	stdio "git.novox.be/novox/mesh-sdk/go"
+
+	"mesh-delivery-builder/internal/build"
+	"mesh-delivery-builder/internal/versions"
+)
+
+// waitsTools are builder_waits, which reads the builder's toolchain waits, and builder_release, which builds an entry with
+// the running build module (design 47 G §276, §672): a hand act, recorded in the hand-act log first, then written to reasks
+// for the builder to answer.
+func waitsTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
+	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
+	return []stdio.Tool{
+		{Name: build.WaitsTool,
+			Description: "The builder's toolchain waits: each entry held because its fingerprint names a toolchain version of its language other than the " +
+				"agreed one, with its module, fingerprint, what it waits on (toolchain:<language>@<version>) and since when. It ends when the agreed toolchain " +
+				"names that version (ran) or when " + build.ReleaseTool + " builds it with the running build module (released). Read-only.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{}},
+			Run: func(map[string]any) (any, error) {
+				keys, err := rr.Keys()
+				if err != nil {
+					return nil, err
+				}
+				sort.Strings(keys)
+				rows := []build.ToolchainWait{}
+				for _, k := range keys {
+					raw, ok, err := rr.Get(k)
+					if err != nil || !ok {
+						continue
+					}
+					var w build.ToolchainWait
+					if json.Unmarshal(raw, &w) == nil {
+						rows = append(rows, w)
+					}
+				}
+				return map[string]any{"waits": rows}, nil
+			}},
+		{Name: build.ReleaseTool,
+			Description: "Build a held entry with the running build module now, whatever toolchain its fingerprint names (a toolchain wait of " + build.WaitsTool +
+				"), and end its wait released. Written in the hand-act log (mesh-hand-acts).",
+			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
+				"module": map[string]any{"type": "string", "description": "the module's name"},
+				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_waits shows it"},
+				"why":    map[string]any{"type": "string", "description": "why, in plain words"}}},
+			Run: func(args map[string]any) (any, error) {
+				key, value, err := build.NewRelease(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
+				if err != nil {
+					return nil, err
+				}
+				if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("released the toolchain wait of %s at %s by hand", str(args, "module"), str(args, "fp")),
+					Why: str(args, "why"), Cause: build.ReleaseTool, Verb: "release"}); err != nil {
+					return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was released: %w", err)
+				}
+				raw, err := ask(key, value)
+				if err != nil {
+					return nil, err
+				}
+				a, err := build.DecodeReAsk(raw)
+				if err != nil {
+					return nil, err
+				}
+				if a.Refused {
+					return nil, errors.New(a.Answer)
+				}
+				return map[string]any{"key": key, "answer": a.Answer}, nil
+			}},
+	}
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
+
+	"mesh-delivery-builder/internal/build"
+	"mesh-delivery-builder/internal/versions"
+)
+
+// Design 47 G §276, §672: builder_waits lists the held entries; builder_release records the hand act first and asks the
+// builder through reasks, whose refusal is the tool's error.
+func TestWaitsToolsListTheWaitsAndReleaseAfterTheHandAct(t *testing.T) {
+	wait, _ := json.Marshal(build.ToolchainWait{Module: "m", Fingerprint: "src2:" + strings.Repeat("1", 64), Language: "go", Version: "go1.26.0", Since: time.Now()})
+	rr := versions.Reader{Keys: func() ([]string, error) { return []string{"m.k"}, nil },
+		Get: func(string) (json.RawMessage, bool, error) { return wait, true, nil }}
+	var asked []build.ReAsk
+	answer := func(key string, value []byte) (json.RawMessage, error) {
+		a, _ := build.DecodeReAsk(value)
+		asked = append(asked, a)
+		a.Answered, a.Answer = time.Now(), "released: built"
+		return json.Marshal(a)
+	}
+	recorded := 0
+	acts := meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { recorded++; return json.RawMessage(`{}`), nil }}
+	tools := waitsTools(rr, answer, time.Now, acts)
+	byName := map[string]func(map[string]any) (any, error){}
+	for _, tl := range tools {
+		byName[tl.Name] = tl.Run
+	}
+	got, err := byName[build.WaitsTool](nil)
+	if rows := got.(map[string]any)["waits"].([]build.ToolchainWait); err != nil || len(rows) != 1 || rows[0].On() != "toolchain:go@go1.26.0" {
+		t.Fatalf("waits = %v, %v", got, err)
+	}
+	if _, err := byName[build.ReleaseTool](map[string]any{"module": "m", "fp": "src2:" + strings.Repeat("1", 64), "why": ""}); err == nil || recorded != 0 {
+		t.Fatalf("a release with no why was taken: %v, %d recorded", err, recorded)
+	}
+	if _, err := byName[build.ReleaseTool](map[string]any{"module": "m", "fp": "src2:" + strings.Repeat("1", 64), "why": "the build module runs"}); err != nil ||
+		recorded != 1 || len(asked) != 1 || asked[0].Mode != build.ModeRelease || asked[0].Module != "m" {
+		t.Fatalf("release: %v, %d recorded, asked %+v", err, recorded, asked)
+	}
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"os/signal"
 	"slices"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -48,6 +49,8 @@ const (
 	// build compiles): from its start, never from its put, so an ask waiting in its stream for a free slot is not failed
 	// (buildrun.Asker.Bound; ADR 0525 §3, §6).
 	defaultStepBound = 30 * time.Minute
+	// defaultToolchainWait is the bound of a toolchain wait, its setting toolchain-wait-minutes (design 47 G §672).
+	defaultToolchainWait = 60 * time.Minute
 )
 
 func main() {
@@ -183,6 +186,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf}
 	b.Left, b.Ended = unput.left, unput.ended
 	b.Numbers = build.VersionNumbers{N: numberer}
+	// The builder's toolchain waits (design 47 G §672): its state waits, and wait-held and wait-ended on its stream waits; its
+	// bound is its setting toolchain-wait-minutes.
+	waitBound := defaultToolchainWait
+	if v := strings.TrimSpace(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES")); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 1 {
+			return fmt.Errorf("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES %q is not a number of minutes", v)
+		}
+		waitBound = time.Duration(n) * time.Minute
+	}
+	waitsBucket := process.AwaitBucket(ctx, js, conn, build.WaitsBucket, "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
+	b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error {
+		return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
+	}, waitBound, logger.Printf)
+	waitsBucket.Ready(func(kv jetstream.KeyValue) { b.Waits.(*waitsKept).start(ctx, kv) })
 	// The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled
 	// and version-released heard; nothing is reserved before the fill.
 	reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf)

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go

@@ -23,6 +23,7 @@ type askAgainer interface {
 	AskAgain(key, why string) []string
 	BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
 	AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
+	Release(ctx context.Context, module, fp, why, by string) string
 }
 
 // buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
@@ -53,6 +54,10 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream
 		announcedAgain(ctx, b, kv, e, r, logf)
 		return
 	}
+	if r.Mode == build.ModeRelease {
+		released(ctx, b, kv, e, r, logf)
+		return
+	}
 	if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop {
 		// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
 		go versionActed(ctx, acts, kv, e, r, logf)
@@ -191,3 +196,21 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
 		logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
 	}
 }
+
+// released answers a reasks value of mode release (the tool builder_release, design 47 G §276, §672): the held entry is built
+// with the running build module, and the value is written back with what was done.
+func released(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
+	if why := b.Release(ctx, r.Module, r.FP, r.Why, r.By); why != "" {
+		r.Answer, r.Refused = why, true
+	} else {
+		r.Answer = "released: the entry is built with the running build module, and its wait ended"
+	}
+	r.Answered = time.Now().UTC()
+	data, err := json.Marshal(r)
+	if err != nil {
+		return
+	}
+	if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
+	}
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/waits.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+
+	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
+	"git.novox.be/novox/mesh-sdk/go/process"
+
+	"mesh-delivery-builder/internal/build"
+)
+
+// waitBody is the body of wait-held and wait-ended on the builder's stream waits (design 47 G §672): the fields of §629's
+// wait with `step` building, `on` toolchain:<language>@<version>, `module` and `fingerprint`, and, on wait-ended, `how`.
+type waitBody struct {
+	Step        string    `json:"step"`
+	On          string    `json:"on"`
+	Module      string    `json:"module"`
+	Fingerprint string    `json:"fingerprint"`
+	Since       time.Time `json:"since"`
+	Bound       string    `json:"bound,omitempty"`
+	How         string    `json:"how,omitempty"`
+}
+
+const waitStepBuilding = "building"
+
+// waitsKept keeps the builder's toolchain waits in its old state waits, one key <module>.<fingerprint> (design 47 G §672),
+// and says wait-held and wait-ended on its old stream waits; at its start it says wait-held again for every key.
+type waitsKept struct {
+	kv    *process.AwaitedBucket
+	say   func(ctx context.Context, event, id string, body any) error
+	bound time.Duration
+	logf  func(string, ...any)
+
+	mu   sync.Mutex
+	held map[string]build.ToolchainWait
+}
+
+func newWaitsKept(kv *process.AwaitedBucket, say func(context.Context, string, string, any) error, bound time.Duration, logf func(string, ...any)) *waitsKept {
+	return &waitsKept{kv: kv, say: say, bound: bound, logf: logf, held: map[string]build.ToolchainWait{}}
+}
+
+func (w *waitsKept) body(t build.ToolchainWait, how string) waitBody {
+	return waitBody{Step: waitStepBuilding, On: t.On(), Module: t.Module, Fingerprint: t.Fingerprint, Since: t.Since, Bound: w.bound.String(), How: how}
+}
+
+// start reads the state whole and says wait-held again for each key.
+func (w *waitsKept) start(ctx context.Context, kv jetstream.KeyValue) {
+	keys, err := kv.Keys(ctx)
+	if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
+		w.logf("%s cannot be read, so the waits of before this start are not said again: %v", build.WaitsState, err)
+		return
+	}
+	for _, k := range keys {
+		e, err := kv.Get(ctx, k)
+		if err != nil {
+			continue
+		}
+		var t build.ToolchainWait
+		if json.Unmarshal(e.Value(), &t) != nil || t.Module == "" {
+			w.logf("%s.%s holds what is not a wait, and is left", build.WaitsState, k)
+			continue
+		}
+		w.mu.Lock()
+		w.held[k] = t
+		w.mu.Unlock()
+		w.said(ctx, "wait-held", t, "")
+	}
+}
+
+func (w *waitsKept) said(ctx context.Context, event string, t build.ToolchainWait, how string) {
+	if err := w.say(ctx, event+"."+t.Module+"."+t.Fingerprint, event+"-"+t.Module+"-"+t.Fingerprint+"-"+t.Since.UTC().Format(time.RFC3339Nano)+how, w.body(t, how)); err != nil {
+		w.logf("saying %s of %s at %s: %v", event, t.Module, t.Fingerprint, err)
+	}
+}
+
+// Hold keeps the wait and says wait-held, once.
+func (w *waitsKept) Hold(ctx context.Context, t build.ToolchainWait) {
+	key, err := builds.BuilderBuildKey(t.Module, t.Fingerprint)
+	if err != nil {
+		w.logf("a wait of %s at %s has no key: %v", t.Module, t.Fingerprint, err)
+		return
+	}
+	w.mu.Lock()
+	if _, had := w.held[key]; had {
+		w.mu.Unlock()
+		return
+	}
+	w.held[key] = t
+	w.mu.Unlock()
+	if kv := w.kv.KV(); kv != nil {
+		data, _ := json.Marshal(t)
+		if _, err := kv.Put(ctx, key, data); err != nil {
+			w.logf("%s.%s could not be written, so the wait is kept in memory only: %v", build.WaitsState, key, err)
+		}
+	}
+	w.said(ctx, "wait-held", t, "")
+}
+
+// End deletes the wait and says wait-ended with how; it answers whether one was held.
+func (w *waitsKept) End(ctx context.Context, module, fp, how string) bool {
+	key, err := builds.BuilderBuildKey(module, fp)
+	if err != nil {
+		return false
+	}
+	w.mu.Lock()
+	t, had := w.held[key]
+	delete(w.held, key)
+	w.mu.Unlock()
+	if !had {
+		return false
+	}
+	if kv := w.kv.KV(); kv != nil {
+		if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+			w.logf("%s.%s could not be deleted: %v", build.WaitsState, key, err)
+		}
+	}
+	w.said(ctx, "wait-ended", t, how)
+	return true
+}

modules/mesh-delivery-builder/internal/build/run.go

@@ -52,7 +52,10 @@ type Builder struct {
 		Held(ctx context.Context, a sdkstep.StepAsk) (bool, error)
 	}
 	Toolchains ToolchainEntries
-	Manifests  Manifests
+	// Waits keeps the toolchain waits of design 47 G §672. Nil holds none: an entry whose fingerprint names another toolchain is
+	// left unbuilt, as before.
+	Waits     Waits
+	Manifests Manifests
 	// Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966,
 	// novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the
 	// builder's next start. Ended is told each key whose end was written.
@@ -129,6 +132,8 @@ type Builder struct {
 	waiting map[string]map[string]chan string
 	// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
 	hands map[string]*handWatch
+	// released are the keys builder_release is building now, whatever toolchain their fingerprint names (§276, §672).
+	released map[string]bool
 	// builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of
 	// the module takes its older one's build out, and the asker drops that build's asks.
 	builds map[string]context.CancelCauseFunc
@@ -466,15 +471,22 @@ func (b *Builder) startOnce(ctx context.Context, key, module string, m meshcatal
 func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) {
 	key, _ := builds.BuilderBuildKey(module, m.Fp2)
 	defer b.handEnded(key)
+	defer b.unrelease(key)
 	plan, planErr, read := b.planOf(ctx, m.Manifest)
 	nothing := read && planErr == nil && len(plan.Tasks) == 0
-	if planErr == nil && !nothing {
+	if planErr == nil && !nothing && !b.isReleased(key) {
 		if why := b.stale(m, plan); why != "" {
+			if w, ok := b.toolchainWait(module, m, plan); ok && b.Waits != nil {
+				b.Waits.Hold(ctx, w)
+			}
 			// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
 			// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
 			b.logf("%s at %s is not built: %s", module, m.Fp2, why)
 			return
 		}
+		if b.Waits != nil {
+			b.Waits.End(ctx, module, m.Fp2, WaitRan) // a toolchain-agreed names the version the fingerprint names
+		}
 	}
 	b.mu.Lock()
 	trunk := b.trunkFP[key]
@@ -679,7 +691,11 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
 			}
 			tc := ag.Toolchain
 			sitting = append(sitting, SittingOutWords(language, ag)...)
-			if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
+			released := false
+			if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
+				released = b.isReleased(key) // builder_release builds it with the running build module (§276)
+			}
+			if want := m.Toolchains[language]; !released && (want.Version != tc.Version || want.Recipe != tc.Recipe) {
 				return builds.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
 			}
 			u := builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}

modules/mesh-delivery-builder/internal/build/waits.go

@@ -0,0 +1,126 @@
+package build
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
+	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
+	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
+	sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
+)
+
+// **The builder's toolchain wait** (design 47 G §672, §267, §276): the builder holds an entry while its fingerprint names a
+// toolchain version of its language other than the one the agreed toolchain of that language names.
+const (
+	// WaitsTool is builder_waits, which reads the waits; ReleaseTool is builder_release, which builds the entry with the
+	// running build module (§276), a hand act written in the hand-act log.
+	WaitsTool   = "builder_waits"
+	ReleaseTool = "builder_release"
+	// WaitsState is the builder's old state section of the waits, one key <module>.<fingerprint>.
+	WaitsState  = "waits"
+	WaitsBucket = builds.BuilderModule + "_" + WaitsState
+	// ModeRelease is the Mode of a reasks value of builder_release.
+	ModeRelease = "release"
+	// WaitRan ends a wait when a toolchain-agreed names the version; WaitReleased when builder_release ended it.
+	WaitRan      = "ran"
+	WaitReleased = "released"
+)
+
+// ToolchainWait is one wait: the entry's module and fingerprint, and the toolchain it waits for.
+type ToolchainWait struct {
+	Module      string    `json:"module"`
+	Fingerprint string    `json:"fingerprint"`
+	Language    string    `json:"language"`
+	Version     string    `json:"version"`
+	Since       time.Time `json:"since"`
+}
+
+// On is what the wait waits for, as the wait's `on` says it.
+func (w ToolchainWait) On() string { return "toolchain:" + w.Language + "@" + w.Version }
+
+// Waits keeps the builder's waits: Hold writes one in the state waits and says wait-held, once per wait; End deletes it and
+// says wait-ended with how, and answers whether one was held.
+type Waits interface {
+	Hold(ctx context.Context, w ToolchainWait)
+	End(ctx context.Context, module, fp, how string) bool
+}
+
+// toolchainWait is the toolchain wait of a module's fingerprint, or false when no language of its plan names a version
+// other than the agreed one (a language with no agreed toolchain is not a wait of §672).
+func (b *Builder) toolchainWait(module string, m meshcatalog.ModulesByModuleValue, plan Plan) (ToolchainWait, bool) {
+	for _, bm := range plan.Modules() {
+		if !plan.NeedsToolchain(bm) {
+			continue
+		}
+		language := buildrun.LanguageOf[bm]
+		ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language))
+		if want, ok := m.Toolchains[language]; ag.Wait == "" && ok && want.Version != ag.Toolchain.Version {
+			return ToolchainWait{Module: module, Fingerprint: m.Fp2, Language: language, Version: want.Version, Since: b.now()}, true
+		}
+	}
+	return ToolchainWait{}, false
+}
+
+// Release builds the held entry of module at fp with the running build module, now, and ends its wait released (design 47
+// G §276, §672). It answers why nothing was done, or empty.
+func (b *Builder) Release(ctx context.Context, module, fp, why, by string) string {
+	b.init()
+	key, err := builds.BuilderBuildKey(module, fp)
+	if err != nil {
+		return err.Error()
+	}
+	if b.Waits == nil {
+		return "this builder keeps no waits"
+	}
+	b.mu.Lock()
+	m, known := b.known[module]
+	b.mu.Unlock()
+	if !known || m.Fp2 != fp || m.Manifest == "" {
+		return fmt.Sprintf("module %s does not have the fingerprint %s now: the builder builds the module's current fingerprint", module, fp)
+	}
+	b.mu.Lock()
+	if b.released == nil {
+		b.released = map[string]bool{}
+	}
+	b.released[key] = true
+	b.mu.Unlock()
+	if !b.Waits.End(ctx, module, fp, WaitReleased) {
+		b.mu.Lock()
+		delete(b.released, key)
+		b.mu.Unlock()
+		return fmt.Sprintf("%s holds no toolchain wait: %s builds a failed key again", key, BuildAgainTool)
+	}
+	b.logf("%s: released by %s (%s): built with the running build module", key, by, why)
+	b.startOnce(ctx, key, module, m)
+	return ""
+}
+
+func (b *Builder) isReleased(key string) bool {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.released[key]
+}
+
+func (b *Builder) unrelease(key string) {
+	b.mu.Lock()
+	delete(b.released, key)
+	b.mu.Unlock()
+}
+
+// NewRelease is the value builder_release writes for a module's fingerprint: its key and the value.
+func NewRelease(module, fp, why, by string, at time.Time) (string, []byte, error) {
+	key, err := builds.BuilderBuildKey(module, fp)
+	if err != nil {
+		return "", nil, err
+	}
+	if strings.TrimSpace(why) == "" {
+		return "", nil, errors.New("say why the entry is released (why)")
+	}
+	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeRelease, Module: module, FP: fp})
+	return key, data, err
+}

modules/mesh-delivery-builder/internal/build/waits_test.go

@@ -0,0 +1,84 @@
+package build
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
+)
+
+// memWaits records the waits held and ended.
+type memWaits struct {
+	mu    sync.Mutex
+	held  map[string]ToolchainWait
+	ended []string
+}
+
+func (w *memWaits) Hold(_ context.Context, t ToolchainWait) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.held == nil {
+		w.held = map[string]ToolchainWait{}
+	}
+	w.held[t.Module+"."+t.Fingerprint] = t
+}
+
+func (w *memWaits) End(_ context.Context, module, fp, how string) bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	k := module + "." + fp
+	if _, ok := w.held[k]; !ok {
+		return false
+	}
+	delete(w.held, k)
+	w.ended = append(w.ended, how)
+	return true
+}
+
+// Design 47 G §672, §267, §276: an entry whose fingerprint names a toolchain version other than the agreed one is held,
+// built and ended `ran` when the agreed toolchain moves to that version; and one held is built with the running build
+// module by Release and ended `released`.
+func TestAnEntryWhoseFingerprintNamesAnotherToolchainWaitsAndIsBuiltWhenItIsAgreedOrReleased(t *testing.T) {
+	old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}}
+	t.Run("ran", func(t *testing.T) {
+		r := newRig(t, goBundle)
+		w := &memWaits{}
+		r.b.Waits = w
+		r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
+		r.b.Wait()
+		if len(w.held) != 1 || len(r.st.vals) != 0 {
+			t.Fatalf("held %v, keys %v: want one wait and nothing written", w.held, r.st.vals)
+		}
+		for _, h := range w.held {
+			if h.On() != "toolchain:go@go1.26.0" {
+				t.Errorf("the wait is on %s", h.On())
+			}
+		}
+		r.tools.Set("ace", tcOf("go", "go1.26.0"))
+		r.tools.Set("novox", tcOf("go", "go1.26.0"))
+		r.b.Reconsider(context.Background())
+		r.b.Wait()
+		if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || len(w.held) != 0 || len(w.ended) != 1 || w.ended[0] != WaitRan {
+			t.Errorf("build %+v, held %v, ended %v: want done and one wait ended ran", v, w.held, w.ended)
+		}
+	})
+	t.Run("released", func(t *testing.T) {
+		r := newRig(t, goBundle)
+		w := &memWaits{}
+		r.b.Waits = w
+		v := moduleValueWith(t, "modules/m", old)
+		r.b.Module(context.Background(), "m", v)
+		r.b.Wait()
+		if why := r.b.Release(context.Background(), "m", fp1, "the build module runs", "operator"); why != "" {
+			t.Fatalf("release refused: %s", why)
+		}
+		r.b.Wait()
+		if got := valueOf(t, r.st, r.key); got.State != builds.BuildDone || got.Toolchains["go"].Version != "go1.27.1" || len(w.ended) != 1 || w.ended[0] != WaitReleased {
+			t.Errorf("build %+v, ended %v: want done with the running toolchain and a wait ended released", got, w.ended)
+		}
+		if why := r.b.Release(context.Background(), "m", fp1, "again", "operator"); why == "" {
+			t.Error("a second release of an entry that is not held was taken")
+		}
+	})
+}

modules/mesh-delivery-builder/manifest_test.go

@@ -50,7 +50,13 @@ type manifest struct {
 	Emits    []string `json:"emits"`
 	Consumes []string `json:"consumes"`
 	State    []state  `json:"state"`
-	Logs     []struct {
+	Streams  []struct {
+		Name       string   `json:"name"`
+		Events     []string `json:"events"`
+		MaxAgeDays int      `json:"max-age-days"`
+		MaxMiB     int      `json:"max-mib"`
+	} `json:"streams"`
+	Logs []struct {
 		Name       string `json:"name"`
 		MaxAgeDays int    `json:"max-age-days"`
 		MaxMiB     int    `json:"max-mib"`
@@ -59,10 +65,13 @@ type manifest struct {
 	Uses    []string `json:"uses"`
 	Invokes []string `json:"invokes"`
 	Config  map[string]struct {
-		Type        string `json:"type"`
-		Description string `json:"description"`
-		Pattern     string `json:"pattern"`
-		Default     string `json:"default"`
+		Type        string  `json:"type"`
+		Description string  `json:"description"`
+		Pattern     string  `json:"pattern"`
+		Minimum     float64 `json:"minimum"`
+		Maximum     float64 `json:"maximum"`
+		Trusted     bool    `json:"trusted"`
+		Default     any     `json:"default"`
 	} `json:"config"`
 	OwnSecrets   map[string]string `json:"own-secrets"`
 	SecretsOwner string            `json:"secrets-owner"`
@@ -102,7 +111,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	if m.Module != "mesh-delivery-builder" {
 		t.Errorf("module = %q", m.Module)
 	}
-	if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
+	if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
 		t.Errorf("state = %+v, want %+v", m.State, want)
 	}
 	// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -134,8 +143,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 		"mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
 		"seat:git.tag",
 		"seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove",
-		"seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 {
-		t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
+		"seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
+		t.Errorf("invokes = %v, want %v", m.Invokes, want)
 	}
 	// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
 	// build module's toolchains; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the
@@ -161,9 +170,16 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	}
 	// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
 	if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
-		"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) {
+		"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) {
 		t.Errorf("tools = %v, want %v", m.Tools, want)
 	}
+	// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
+	if len(m.Streams) != 1 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 {
+		t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB", m.Streams)
+	}
+	if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 1 {
+		t.Errorf("config = %+v, want toolchain-wait-minutes of 60", m.Config)
+	}
 	// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
 	if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
 		t.Errorf("emits = %v, want %v", m.Emits, want)
@@ -176,7 +192,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 	if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved"}) {
 		t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved", m.Claims)
 	}
-	for _, k := range []string{"streams", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
+	for _, k := range []string{"provides", "requires", "defines-seats", "seat", "seats", "bus"} {
 		if _, ok := top[k]; ok {
 			t.Errorf("the manifest declares %q: the builder has none", k)
 		}
@@ -202,8 +218,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
 		t.Errorf("process = %v", p)
 	}
 	env, _ := p["env"].(map[string]any)
-	if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
-		t.Errorf("env = %v, want the broker file alone", p["env"])
+	if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES"] != "${config:toolchain-wait-minutes}" {
+		t.Errorf("env = %v, want the broker file and the toolchain wait's setting (design 47 G §672)", p["env"])
 	}
 	for _, r := range m.Resources {
 		if r["id"] == "settings" {

modules/mesh-delivery-builder/module.json

@@ -23,11 +23,13 @@
     "builder_old_form",
     "builder_publish_again",
     "builder_reserved",
+    "builder_release",
     "builder_reserved_drop",
     "builder_seat_queue",
     "builder_stuck",
     "builder_toolchains",
     "builder_toolchains_drop",
+    "builder_waits",
     "builder_versions",
     "reserved",
     "builder_first_views",
@@ -107,11 +109,25 @@
     { "name": "bounds", "history": 1, "key-ttl": true },
     { "name": "queues", "history": 1 },
     { "name": "toolchains-heard", "history": 1 },
+    { "name": "waits", "history": 1 },
     { "name": "reserved", "history": 1 },
     { "name": "reasks", "history": 1 },
     { "name": "versions", "history": 1 },
     { "name": "first-views", "history": 1 }
   ],
+  "streams": [
+    { "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 }
+  ],
+  "config": {
+    "toolchain-wait-minutes": {
+      "type": "number",
+      "description": "minutes an entry waits for the agreed toolchain of its language to name the version its fingerprint names (design 47 G section 672); a bound a whole test sets in seconds.",
+      "minimum": 1,
+      "maximum": 10080,
+      "trusted": false,
+      "default": 60
+    }
+  },
   "logs": [
     { "name": "build-log", "max-age-days": 30, "max-mib": 3072 }
   ],
@@ -156,7 +172,8 @@
         "./mesh-delivery-builder"
       ],
       "env": {
-        "MESH_BROKER_FILE": "${dir:state}/broker"
+        "MESH_BROKER_FILE": "${dir:state}/broker",
+        "MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES": "${config:toolchain-wait-minutes}"
       },
       "user": "mesh-delivery-builder",
       "health": {