Hold the build seat's checks while a delivery builds, and let one check that waited 30 minutes through

ADR 0483: the controller keeps a key per plan in its build phase; holders read it and
take no new check while one is there, but for the oldest check past 30 minutes, one
at a time by a let-through key. The queue says why checks wait.

cmd/mesh-controller/plan_looks.go

@@ -206,6 +206,7 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
 	return func(_ *inventory.Inventory, p inventory.Plan) {
 		sayPlanMoved(ctx, bus, open.inventory, p)
 		planKept(p)
+		keepBuildingKey(ctx, p)
 		if p.WalkSaid == inventory.WalkEnded {
 			go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
 		}

cmd/mesh-controller/plans_building.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
+//
+// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
+// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
+// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
+// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
+// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
+// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
+//
+// The layout is documented in internal/link/holds.go.
+
+// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
+// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
+// it is in the phase still, which is what keeps a check from taking a place between two tiers.
+func buildPhaseOf(p inventory.Plan) bool {
+	if !p.Open() || p.Waiting() || p.Release != nil {
+		return false
+	}
+	asked, unbuilt := false, false
+	for _, tier := range p.Tiers {
+		for _, m := range tier {
+			st := p.Modules[m]
+			if st != nil && st.State != "" {
+				asked = true
+			}
+			if st == nil || st.State != "built" {
+				unbuilt = true
+			}
+		}
+	}
+	return asked && unbuilt
+}
+
+// buildingSince is when the plan's first build was asked, or fallback when no module says.
+func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
+	since := time.Time{}
+	for _, st := range p.Modules {
+		if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
+			since = *st.AskedAt
+		}
+	}
+	if since.IsZero() {
+		return fallback
+	}
+	return since
+}
+
+// buildingStore is the bucket of the plans building.
+type buildingStore interface {
+	Put(ctx context.Context, plan string, value []byte) error
+	Delete(ctx context.Context, plan string) error
+	Keys(ctx context.Context) ([]string, error)
+}
+
+// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
+// other plan, deleted. It returns the faults, each once.
+func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
+	var faults []error
+	live := map[string]bool{}
+	for _, p := range plans {
+		if !buildPhaseOf(p) {
+			continue
+		}
+		live[p.ID] = true
+		if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
+			faults = append(faults, err)
+		}
+	}
+	keys, err := store.Keys(ctx)
+	if err != nil {
+		return append(faults, err)
+	}
+	for _, key := range keys {
+		if live[key] {
+			continue
+		}
+		if err := store.Delete(ctx, key); err != nil {
+			faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
+		}
+	}
+	return faults
+}
+
+func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
+	body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
+	if err := store.Put(ctx, plan, body); err != nil {
+		return fmt.Errorf("the key of %s could not be put: %w", plan, err)
+	}
+	return nil
+}
+
+// busBuilding is the store on the controller's bucket.
+type busBuilding struct {
+	conn *nats.Conn
+	mu   sync.Mutex
+	kv   jetstream.KeyValue
+}
+
+func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.kv != nil {
+		return b.kv, nil
+	}
+	api, err := jetstream.New(b.conn)
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
+	if err != nil {
+		return nil, fmt.Errorf("the plans building: %w", err)
+	}
+	b.kv = kv
+	return kv, nil
+}
+
+func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
+	kv, err := b.bucket(ctx)
+	if err != nil {
+		return err
+	}
+	_, err = kv.Put(ctx, plan, value)
+	return err
+}
+
+func (b *busBuilding) Delete(ctx context.Context, plan string) error {
+	kv, err := b.bucket(ctx)
+	if err != nil {
+		return err
+	}
+	if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
+	kv, err := b.bucket(ctx)
+	if err != nil {
+		return nil, err
+	}
+	keys, err := kv.Keys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, nil
+	}
+	return keys, err
+}
+
+// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
+var buildingKeys atomic.Pointer[buildingStore]
+
+func setBuildingStore(s buildingStore) {
+	if s == nil {
+		buildingKeys.Store(nil)
+		return
+	}
+	buildingKeys.Store(&s)
+}
+
+// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
+// said; the look that follows puts it again.
+func sayBuilding(ctx context.Context, plan string, since time.Time) {
+	s := buildingKeys.Load()
+	if s == nil {
+		return
+	}
+	if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
+		fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
+	}
+}
+
+// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
+func keepBuildingKey(ctx context.Context, p inventory.Plan) {
+	s := buildingKeys.Load()
+	if s == nil {
+		return
+	}
+	if buildPhaseOf(p) {
+		sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
+		return
+	}
+	if err := (*s).Delete(ctx, p.ID); err != nil {
+		fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
+	}
+}
+
+// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
+// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
+func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
+	store := &busBuilding{conn: conn}
+	setBuildingStore(store)
+	defer setBuildingStore(nil)
+	tick := time.NewTicker(broker.PlansBuildingRefresh)
+	defer tick.Stop()
+	failing := ""
+	for {
+		if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
+			why := ""
+			if plans, err := inv.OpenPlans(ctx); err != nil {
+				why = fmt.Sprintf("the plans cannot be read: %v", err)
+			} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
+				why = fmt.Sprint(faults)
+			}
+			if why != failing {
+				if why != "" {
+					fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
+				} else {
+					fmt.Printf("plans: the keys of the plans building are kept again\n")
+				}
+				failing = why
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+	}
+}

cmd/mesh-controller/plans_building_test.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// Issue 1687 (ADR 0483 clause 2): a delivery plan's key lives from the ask of its first build until every build of its last
+// tier is built, or the plan fails, stops or waits on the operator.
+
+type fakeBuildingStore struct {
+	mu   sync.Mutex
+	keys map[string]link.PlanBuilding
+	puts map[string]int
+}
+
+func newFakeBuildingStore() *fakeBuildingStore {
+	return &fakeBuildingStore{keys: map[string]link.PlanBuilding{}, puts: map[string]int{}}
+}
+
+func (f *fakeBuildingStore) Put(_ context.Context, plan string, value []byte) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var v link.PlanBuilding
+	if err := json.Unmarshal(value, &v); err != nil {
+		return err
+	}
+	f.keys[plan] = v
+	f.puts[plan]++
+	return nil
+}
+
+func (f *fakeBuildingStore) Delete(_ context.Context, plan string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	delete(f.keys, plan)
+	return nil
+}
+
+func (f *fakeBuildingStore) Keys(context.Context) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for k := range f.keys {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+func aPlan(id, state string, tiers [][]string, states map[string]string) inventory.Plan {
+	p := inventory.Plan{ID: id, State: state, Tiers: tiers, Modules: map[string]*inventory.PlanModule{}}
+	now := time.Now().Add(-time.Minute)
+	for m, st := range states {
+		pm := &inventory.PlanModule{State: st}
+		if st != "" {
+			pm.AskedAt = &now
+		}
+		p.Modules[m] = pm
+	}
+	return p
+}
+
+func TestAPlanIsInItsBuildPhaseFromTheFirstAskToTheLastBuild(t *testing.T) {
+	tiers := [][]string{{"a", "b"}, {"c"}}
+	cases := []struct {
+		name string
+		plan inventory.Plan
+		want bool
+	}{
+		{"no build asked yet", aPlan("p", inventory.PlanBuilding, tiers, nil), false},
+		{"the first tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked", "b": "asked"}), true},
+		// Between two tiers: the first built and sent (rolling), the second not asked yet.
+		{"between two tiers", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"}), true},
+		{"the last tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "built", "b": "built", "c": "asked"}), true},
+		{"every build built", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": "built"}), false},
+		{"failed", aPlan("p", inventory.PlanFailed, tiers, map[string]string{"a": "failed", "b": "asked"}), false},
+		{"superseded", aPlan("p", inventory.PlanSuperseded, tiers, map[string]string{"a": "asked"}), false},
+		{"a batch whose window is open", aPlan("p", inventory.PlanAssembling, tiers, map[string]string{"a": "asked"}), false},
+	}
+	for _, c := range cases {
+		if got := buildPhaseOf(c.plan); got != c.want {
+			t.Errorf("%s: in the build phase = %v, want %v", c.name, got, c.want)
+		}
+	}
+	waiting := aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
+	waiting.Delivery = &inventory.PlanDelivery{Awaits: "the operator's word"}
+	if buildPhaseOf(waiting) {
+		t.Error("a plan that waits on the operator is in its build phase")
+	}
+}
+
+// The key's life: put, refreshed, deleted at the phase's end; a plan that left the phase unseen loses its key at the next look.
+func TestAPlanKeyIsPutRefreshedAndDeletedAtThePhasesEnd(t *testing.T) {
+	ctx := context.Background()
+	store := newFakeBuildingStore()
+	tiers := [][]string{{"a"}, {"b"}}
+	building := aPlan("plan-1", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
+
+	if faults := syncBuilding(ctx, store, []inventory.Plan{building}, time.Now()); len(faults) > 0 {
+		t.Fatal(faults)
+	}
+	if _, ok := store.keys["plan-1"]; !ok {
+		t.Fatal("no key while the first build is asked")
+	}
+	first := store.keys["plan-1"]
+	// Refreshed: put again, the phase's start kept.
+	time.Sleep(2 * time.Millisecond)
+	syncBuilding(ctx, store, []inventory.Plan{building}, time.Now())
+	if store.puts["plan-1"] != 2 || !store.keys["plan-1"].Seen.After(first.Seen) || !store.keys["plan-1"].Since.Equal(first.Since) {
+		t.Errorf("the key was not refreshed with its start kept: %d puts, %+v then %+v", store.puts["plan-1"], first, store.keys["plan-1"])
+	}
+	// Between the tiers: still there.
+	between := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built"})
+	syncBuilding(ctx, store, []inventory.Plan{between}, time.Now())
+	if _, ok := store.keys["plan-1"]; !ok {
+		t.Error("the key was gone between two tiers")
+	}
+	// A key of another plan that is no longer in the phase.
+	store.keys["plan-0"] = link.PlanBuilding{Plan: "plan-0"}
+	done := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"})
+	syncBuilding(ctx, store, []inventory.Plan{done}, time.Now())
+	if len(store.keys) != 0 {
+		t.Errorf("keys remain after the last build was built: %v", store.keys)
+	}
+}
+
+// keepBuildingKey, told of each save, puts the key while the plan builds and deletes it when the plan fails.
+func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
+	ctx := context.Background()
+	store := newFakeBuildingStore()
+	var s buildingStore = store
+	buildingKeys.Store(&s)
+	defer buildingKeys.Store(nil)
+
+	p := aPlan("plan-7", inventory.PlanBuilding, [][]string{{"a"}}, map[string]string{"a": "asked"})
+	keepBuildingKey(ctx, p)
+	if _, ok := store.keys["plan-7"]; !ok {
+		t.Fatal("the key was not put when the plan was saved building")
+	}
+	p.State = inventory.PlanFailed
+	keepBuildingKey(ctx, p)
+	if _, ok := store.keys["plan-7"]; ok {
+		t.Fatal("the key stayed after the plan failed")
+	}
+}
+
+// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
+func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
+	store := newFakeBuildingStore()
+	var s buildingStore = store
+	buildingKeys.Store(&s)
+	defer buildingKeys.Store(nil)
+	p, byName := aTierOfThree()
+	keyThere := false
+	was := askABuild
+	askABuild = func(context.Context, buildSource, string, string) (string, error) {
+		_, keyThere = store.keys["plan-1"]
+		return "build-x", nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	askModule(t.Context(), p, "a", byName)
+	if !keyThere {
+		t.Fatal("the plan's key was not there when its first build was asked")
+	}
+}

cmd/mesh-controller/push.go

@@ -284,6 +284,8 @@ func serve(ctx context.Context) (err error) {
 	go mirroringRecords(ctx, inv, bus.Conn, bus)
 	// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
 	go sweepingDeclarationFiles(ctx, inv, bus.Conn)
+	// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
+	go keepingPlansBuilding(ctx, inv, bus.Conn)
 	// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
 	// to-be 45 §6).
 	link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })

cmd/mesh-controller/queue.go

@@ -91,7 +91,9 @@ type queueAnswer struct {
 	Dead       int            `json:"dead"`
 	MaxDeliver int            `json:"max-deliver"`
 	Asks       []queuedAnswer `json:"asks"`
-	Text       string         `json:"text"`
+	// HeldFor are the delivery plans building, while which the waiting checks are held (ADR 0483 clause 5); empty when none.
+	HeldFor []string `json:"held-for,omitempty"`
+	Text    string   `json:"text"`
 }
 
 // queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
@@ -120,6 +122,9 @@ type queuedAnswer struct {
 	// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
 	// (novox/hq issue 594).
 	Delivery int `json:"delivery,omitempty"`
+	// LetThrough is a check that waited past 30 minutes and is let through while a delivery builds: the next one, or the one
+	// that runs (ADR 0483 clause 5).
+	LetThrough bool `json:"let-through,omitempty"`
 }
 
 // aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
@@ -130,7 +135,8 @@ var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
 func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
 	waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
 	out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
-		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
+		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, HeldFor: q.Hold.Plans, Text: queueText(q, now)}
+	letNext, letRunning := q.LetThroughOf(now)
 	one := func(a link.QueuedAsk, position int) queuedAnswer {
 		kind := link.KindOf(a.Request)
 		since := a.AskedAt
@@ -143,7 +149,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
 		}
 		return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
 			Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
-			Position: position, Seq: a.Seq, Delivery: a.Delivery}
+			Position: position, Seq: a.Seq, Delivery: a.Delivery, LetThrough: a.ID != "" && (a.ID == letNext || a.ID == letRunning)}
 	}
 	for _, a := range running {
 		out.Asks = append(out.Asks, one(a, 0))
@@ -179,6 +185,11 @@ func queueText(q link.Queue, now time.Time) string {
 		}
 		return s
 	}
+	letNext, letRunning := q.LetThroughOf(now)
+	const passed = "waited past 30 minutes, let through"
+	if q.HoldFault != "" {
+		fmt.Fprintf(&b, "why checks wait cannot be read, so none is said held: %s\n", q.HoldFault)
+	}
 	if len(running) > 0 {
 		fmt.Fprintln(&b, "\nin flight:")
 		for _, a := range running {
@@ -193,14 +204,27 @@ func queueText(q link.Queue, now time.Time) string {
 			case a.Was != "":
 				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
 			}
-			fmt.Fprintf(&b, "  %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
-				ago(a.AskedAt), a.Seq)
+			note := ""
+			if a.ID != "" && a.ID == letRunning {
+				note = " — " + passed
+			}
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s, %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), where,
+				ago(a.AskedAt), a.Seq, note)
 		}
 	}
 	if len(waiting) > 0 {
 		fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
+		// **Why the checks wait** (ADR 0483 clause 5): above them, the plans they are held for.
+		if len(q.Hold.Plans) > 0 && hasCheck(waiting) {
+			fmt.Fprintf(&b, "  checks are held while delivery %s builds; the oldest check that waited 30 minutes goes first, one at a time\n",
+				strings.Join(q.Hold.Plans, ", "))
+		}
 		for _, a := range waiting {
-			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
+			note := ""
+			if a.ID != "" && a.ID == letNext {
+				note = " — " + passed + " next"
+			}
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq, note)
 		}
 	}
 	if len(dead) > 0 {
@@ -218,6 +242,16 @@ func queueText(q link.Queue, now time.Time) string {
 	return b.String()
 }
 
+// hasCheck says a check is among the asks.
+func hasCheck(asks []link.QueuedAsk) bool {
+	for _, a := range asks {
+		if a.Request.Check != nil {
+			return true
+		}
+	}
+	return false
+}
+
 // cancelCommand drops one waiting or dead ask.
 func cancelCommand(ctx context.Context, args []string) error {
 	if len(args) != 1 {

cmd/mesh-controller/queue_held_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// Issue 1688 / 1652 AC5 (ADR 0483 clause 5): the queue names, above the waiting checks, the plan they are held for, and says of a
+// check let through that it waited past 30 minutes.
+
+func aHeldQueue(now time.Time) link.Queue {
+	check := func(id string, ago time.Duration, n int) link.QueuedAsk {
+		return link.QueuedAsk{Seq: uint64(n), ID: id, Repository: "ssh://forge/novox/r", AskedAt: now.Add(-ago), State: link.AskWaiting,
+			Request: link.BuildRequest{ID: id, Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: n}}}
+	}
+	return link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{
+		check("build-young", 5*time.Minute, 1),
+		check("build-old", 45*time.Minute, 2),
+		check("build-older", 50*time.Minute, 3),
+	}, Hold: link.CheckHold{Plans: []string{"plan-42"}}}
+}
+
+func TestTheQueueNamesThePlanTheChecksAreHeldFor(t *testing.T) {
+	now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	text := queueText(aHeldQueue(now), now)
+	heldAt := strings.Index(text, "held while delivery plan-42 builds")
+	firstAt := strings.Index(text, "build-young")
+	if heldAt < 0 || firstAt < 0 || heldAt > firstAt {
+		t.Fatalf("the plan the checks are held for is not named above the waiting checks:\n%s", text)
+	}
+	// Nothing building: nothing held.
+	q := aHeldQueue(now)
+	q.Hold = link.CheckHold{}
+	if strings.Contains(queueText(q, now), "held while") {
+		t.Fatal("checks are said held with no delivery building")
+	}
+}
+
+func TestACheckLetThroughIsSaidToHaveWaitedPastThirtyMinutes(t *testing.T) {
+	now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	q := aHeldQueue(now)
+	text := queueText(q, now)
+	var saidOf []string
+	for _, l := range strings.Split(text, "\n") {
+		if strings.Contains(l, "waited past 30 minutes, let through") {
+			saidOf = append(saidOf, strings.Fields(l)[0])
+		}
+	}
+	if len(saidOf) != 1 || saidOf[0] != "build-older" {
+		t.Fatalf("the check let through next (the oldest) is not the one said: %v\n%s", saidOf, text)
+	}
+	answer := queueAnswerOf(q, now)
+	if len(answer.HeldFor) != 1 || answer.HeldFor[0] != "plan-42" {
+		t.Errorf("held-for = %v", answer.HeldFor)
+	}
+	for _, a := range answer.Asks {
+		if a.LetThrough != (a.ID == "build-older") {
+			t.Errorf("%s let-through = %v", a.ID, a.LetThrough)
+		}
+	}
+	// A check in flight under the let-through key is said too.
+	q.Asks = append(q.Asks, link.QueuedAsk{Seq: 9, ID: "build-running", State: link.AskInFlight, On: "ace", AskedAt: now.Add(-time.Hour),
+		Started: now.Add(-time.Minute), Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: 9}}})
+	q.Hold.LetThrough = &link.LetThrough{Builder: "ace"}
+	if !strings.Contains(queueText(q, now), "build-running") || !strings.Contains(queueText(q, now), "waited past 30 minutes, let through\n") {
+		t.Errorf("the check in flight let through is not said:\n%s", queueText(q, now))
+	}
+	// Younger than the bound: none is said.
+	young := aHeldQueue(now)
+	young.Asks = young.Asks[:1]
+	if strings.Contains(queueText(young, now), "waited past") {
+		t.Error("a check of 5 minutes is said to have waited past 30")
+	}
+}

cmd/mesh-controller/release_plan.go

@@ -577,6 +577,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
 	}
 	// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
 	// requests it answers, whose checks name the part of a differing key.
+	// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
+	// would hold a place for its whole run.
+	sayBuilding(ctx, p.ID, now)
 	id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
 		e.Source.Path, ref)
 	if err != nil {