cmd/mesh-controller/plan_looks.go

@@ -0,0 +1,568 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on
+// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a
+// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for
+// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are
+// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is
+// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be
+// (armWalkEnd).
+
+// looker runs the look at the plans that events ask for.
+type looker struct {
+	signal chan struct{} // one place: asks made together are one look
+	look   func(ctx context.Context) (failed bool)
+	// gap is the least time between two looks, so a plan that kept changing could not make the worker spin.
+	gap time.Duration
+	// first runs before the first look: what a controller that took the lease sets up.
+	first func(ctx context.Context)
+
+	mu      sync.Mutex
+	retry   *time.Timer
+	backoff time.Duration
+	base    time.Duration
+	most    time.Duration
+}
+
+const (
+	retryLookFirst = 5 * time.Second
+	retryLookMost  = time.Minute
+	lookGap        = 200 * time.Millisecond
+)
+
+func newLooker(look func(ctx context.Context) bool) *looker {
+	return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost}
+}
+
+// ask has a look run, soon; never blocks.
+func (l *looker) ask() {
+	select {
+	case l.signal <- struct{}{}:
+	default:
+	}
+}
+
+// run takes the asks one at a time until the context ends.
+func (l *looker) run(ctx context.Context) {
+	if l.first != nil {
+		l.first(ctx)
+	}
+	for {
+		select {
+		case <-ctx.Done():
+			l.mu.Lock()
+			if l.retry != nil {
+				l.retry.Stop()
+			}
+			l.mu.Unlock()
+			return
+		case <-l.signal:
+		}
+		l.after(l.look(ctx))
+		if l.gap > 0 {
+			select {
+			case <-ctx.Done():
+			case <-time.After(l.gap):
+			}
+		}
+	}
+}
+
+// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures.
+func (l *looker) after(failed bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if !failed {
+		l.backoff = 0
+		if l.retry != nil {
+			l.retry.Stop()
+			l.retry = nil
+		}
+		return
+	}
+	if l.retry != nil {
+		return // one is armed already
+	}
+	switch {
+	case l.backoff == 0:
+		l.backoff = l.base
+	case l.backoff*2 > l.most:
+		l.backoff = l.most
+	default:
+		l.backoff *= 2
+	}
+	// retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by
+	// the first look that fails nothing; the step is otherwise started by an event, and a failure is none.
+	l.retry = time.AfterFunc(l.backoff, func() {
+		l.mu.Lock()
+		l.retry = nil
+		l.mu.Unlock()
+		l.ask()
+	})
+}
+
+// theLooker is the serving controller's; nil in a command, whose looks are its own.
+var (
+	lookerMu  sync.RWMutex
+	theLooker *looker
+)
+
+// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send,
+// a silent key expired. Nothing in a process that is not serving.
+func askToLook() {
+	lookerMu.RLock()
+	l := theLooker
+	lookerMu.RUnlock()
+	if l != nil {
+		l.ask()
+	}
+}
+
+// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
+// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
+func startPlanLooks(ctx context.Context, open *stores) {
+	l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) })
+	l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) }
+	lookerMu.Lock()
+	theLooker = l
+	lookerMu.Unlock()
+	go func() {
+		<-ctx.Done()
+		lookerMu.Lock()
+		if theLooker == l {
+			theLooker = nil
+		}
+		lookerMu.Unlock()
+	}()
+	go l.run(ctx)
+	l.ask()
+}
+
+// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open.
+const planLookPasses = 6
+
+// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is
+// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the
+// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is
+// answered once.
+func lookAtPlans(ctx context.Context, open *stores) (failed bool) {
+	release, err := open.inventory.HoldPlans(ctx, true)
+	if err != nil {
+		fmt.Printf("plans: cannot hold them: %v\n", err)
+		return true
+	}
+	func() {
+		defer release()
+		// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
+		if _, err := releaseBacklog(ctx, open, ""); err != nil {
+			fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
+			failed = true
+		}
+		// Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a
+		// batch cut makes one.
+		for pass := 0; pass < planLookPasses; pass++ {
+			before := openPlansNow(ctx, open.inventory)
+			if advanceHeld(ctx, open) {
+				failed = true
+			}
+			if openPlansNow(ctx, open.inventory) == before {
+				break
+			}
+		}
+		if err := syncSilentKeys(ctx, open); err != nil {
+			fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err)
+		}
+	}()
+	keepWalkPhases(ctx, open)
+	return failed
+}
+
+// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a
+// single step rather than the look (the tests of the steps).
+func advancePlans(ctx context.Context, open *stores) {
+	release, err := open.inventory.HoldPlans(ctx, false)
+	if err != nil {
+		if !errors.Is(err, inventory.ErrPlansBusy) {
+			fmt.Printf("plans: cannot hold them: %v\n", err)
+		}
+		return
+	}
+	defer release()
+	if _, err := releaseBacklog(ctx, open, ""); err != nil {
+		fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
+	}
+	advanceHeld(ctx, open)
+}
+
+// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not.
+func openPlansNow(ctx context.Context, inv *inventory.Inventory) string {
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return "unreadable " + time.Now().String()
+	}
+	ids := make([]string, 0, len(plans))
+	for _, p := range plans {
+		ids = append(ids, p.ID+":"+string(p.State))
+	}
+	sort.Strings(ids)
+	return strings.Join(ids, ",")
+}
+
+// --- the wait for a machine's report, as keys on the bus ---
+
+var (
+	silentMu    sync.RWMutex
+	silentKeyed silentClock // nil in a process that is not the serving controller
+)
+
+// setSilentClock makes clock the rollout's; nil takes it away.
+func setSilentClock(c silentClock) {
+	silentMu.Lock()
+	defer silentMu.Unlock()
+	silentKeyed = c
+}
+
+func silentClockOf() silentClock {
+	silentMu.RLock()
+	defer silentMu.RUnlock()
+	return silentKeyed
+}
+
+// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent
+// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine
+// judges, and the machines a release's gates wait on. Pure.
+func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time {
+	waits := map[silentWait]time.Time{}
+	put := func(w silentWait, at time.Time) {
+		if at.IsZero() {
+			return
+		}
+		if was, ok := waits[w]; !ok || at.Before(was) {
+			waits[w] = at
+		}
+	}
+	if !p.Open() {
+		return waits
+	}
+	if r := p.Release; r != nil {
+		gates := []*inventory.PlanGate{r.Gate}
+		for _, g := range r.Gates {
+			gates = append(gates, g)
+		}
+		for _, g := range gates {
+			if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil {
+				continue
+			}
+			for _, node := range g.Machines {
+				put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since)
+			}
+		}
+		return waits
+	}
+	if p.Waiting() {
+		return waits
+	}
+	tierOf := func(module string) int {
+		for i, tier := range p.Tiers {
+			if slices.Contains(tier, module) {
+				return i
+			}
+		}
+		return 0
+	}
+	waiting, _, _ := walkReports(p, reports, now, prov)
+	for module, s := range p.Modules {
+		if s == nil {
+			continue
+		}
+		for _, node := range waiting {
+			sent, ok := s.Rest[node]
+			if !ok {
+				continue
+			}
+			at := sent.At
+			if at.IsZero() && s.SentAt != nil {
+				at = *s.SentAt
+			}
+			put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at)
+		}
+		if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil {
+			for _, node := range g.Machines {
+				put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since)
+			}
+		}
+	}
+	return waits
+}
+
+// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the
+// look it wakes finds the machine silent and not a fraction early.
+func silentTTL(sent, now time.Time) time.Duration {
+	left := sent.Add(inventory.ApplySilentAfter).Sub(now) + time.Second
+	if left < time.Second {
+		return time.Second
+	}
+	return left
+}
+
+// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the
+// key is of an earlier send) and the keys to delete (the wait is over). Pure.
+func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) {
+	for w, sent := range want {
+		if was, ok := held[w]; !ok || !was.Equal(sent) {
+			arm = append(arm, w)
+		}
+	}
+	for w := range held {
+		if _, ok := want[w]; !ok {
+			drop = append(drop, w)
+		}
+	}
+	return arm, drop
+}
+
+// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out
+// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the
+// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the
+// look alone knows the report was on that send). Only the lease holder looks, so only it writes.
+func syncSilentKeys(ctx context.Context, open *stores) error {
+	c := silentClockOf()
+	inv := open.inventory
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return err
+	}
+	reports, err := inv.LastReports(ctx)
+	if err != nil {
+		return err
+	}
+	now := time.Now().UTC()
+	want := map[silentWait]time.Time{}
+	for _, p := range plans {
+		for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) {
+			want[w] = at
+		}
+	}
+	if c == nil {
+		// No bucket to put them in: each wait is looked at by a timer of its own, once.
+		for w, sent := range want {
+			silentByTimer(w, sent, silentTTL(sent, now))
+		}
+		return nil
+	}
+	held, err := c.Held(ctx)
+	if err != nil {
+		return err
+	}
+	arm, drop := silentDiff(want, held)
+	var first error
+	for _, w := range arm {
+		if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
+			silentByTimer(w, want[w], silentTTL(want[w], now))
+			if first == nil {
+				first = err
+			}
+		}
+	}
+	for _, w := range drop {
+		if err := c.Reported(ctx, w); err != nil && first == nil {
+			first = err
+		}
+	}
+	return first
+}
+
+var silentTimers = struct {
+	mu     sync.Mutex
+	timers map[silentWait]time.Time
+}{timers: map[silentWait]time.Time{}}
+
+// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
+// (DegradableGrants says why that can be): never silently never. Once for each send.
+func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
+	silentTimers.mu.Lock()
+	defer silentTimers.mu.Unlock()
+	if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
+		return
+	}
+	silentTimers.timers[w] = sent
+	// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
+	time.AfterFunc(ttl, func() {
+		silentTimers.mu.Lock()
+		if silentTimers.timers[w].Equal(sent) {
+			delete(silentTimers.timers, w)
+		}
+		silentTimers.mu.Unlock()
+		askToLook()
+	})
+}
+
+// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
+// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires.
+func startRolloutSilent(ctx context.Context, js *broker.JetStream) error {
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return err
+	}
+	clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() })
+	if err != nil {
+		return err
+	}
+	setSilentClock(clock)
+	go func() {
+		<-ctx.Done()
+		setSilentClock(nil)
+	}()
+	return nil
+}
+
+// --- the end of a walk whose end is not known ---
+
+var walkEnds = struct {
+	mu     sync.Mutex
+	timers map[string]*time.Timer
+}{timers: map[string]*time.Timer{}}
+
+// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded
+// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at
+// each save of the done walk, so it counts from the last.
+func armWalkEnd(id string, savedAt time.Time) {
+	d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second))
+	if d <= 0 {
+		return
+	}
+	walkEnds.mu.Lock()
+	defer walkEnds.mu.Unlock()
+	if t := walkEnds.timers[id]; t != nil {
+		t.Stop()
+	}
+	// retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first
+	// look after the lease was taken arms it again for the walks still unrecorded.
+	walkEnds.timers[id] = time.AfterFunc(d, func() {
+		walkEnds.mu.Lock()
+		delete(walkEnds.timers, id)
+		walkEnds.mu.Unlock()
+		askToLook()
+	})
+}
+
+// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease.
+func rearmWalkEnds(ctx context.Context, open *stores) {
+	recent, err := open.inventory.RecentPlans(ctx, 30)
+	if err != nil {
+		fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err)
+		return
+	}
+	for _, p := range recent {
+		if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour {
+			armWalkEnd(p.ID, p.Updated)
+		}
+	}
+}
+
+// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or
+// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded.
+func planKept(p inventory.Plan) {
+	if p.State == inventory.PlanDone && p.Release == nil {
+		armWalkEnd(p.ID, time.Now())
+	}
+	askToLook()
+}
+
+// --- `sent`: a tier sent to a machine, said ---
+
+// sendMade is one machine a plan's tier was sent to.
+type sendMade struct {
+	Tier    int
+	Node    string
+	Modules []string
+	At      time.Time
+}
+
+// sendsMade collects the sends one step of a plan made, to be said once the plan is saved.
+type sendsMade struct {
+	mu   sync.Mutex
+	list []sendMade
+}
+
+type sendsMadeKey struct{}
+
+func withSendsMade(ctx context.Context, s *sendsMade) context.Context {
+	return context.WithValue(ctx, sendsMadeKey{}, s)
+}
+
+// noteSends records that modules of the plan's tier were sent to the nodes.
+func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) {
+	s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade)
+	if s == nil {
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, n := range nodes {
+		s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at})
+	}
+}
+
+// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine
+// and the modules it was sent, and when.
+type planSentBody struct {
+	Event   string    `json:"event"`
+	Plan    string    `json:"plan"`
+	Tier    int       `json:"tier"`
+	Node    string    `json:"node"`
+	Modules []string  `json:"modules"`
+	At      time.Time `json:"at"`
+}
+
+// sentBodies are the events a step's sends are said as.
+func sentBodies(plan string, sends *sendsMade) []planSentBody {
+	sends.mu.Lock()
+	defer sends.mu.Unlock()
+	out := make([]planSentBody, 0, len(sends.list))
+	for _, s := range sends.list {
+		out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node,
+			Modules: s.Modules, At: s.At})
+	}
+	return out
+}
+
+// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the
+// controller's own next step does not wait for it.
+func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) {
+	bus := checkEvents
+	if bus == nil {
+		return
+	}
+	for _, body := range sentBodies(p.ID, sends) {
+		raw, err := json.Marshal(body)
+		if err != nil {
+			continue
+		}
+		go func() {
+			stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+			defer stop()
+			if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil {
+				fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err)
+			}
+		}()
+	}
+}

cmd/mesh-controller/plan_looks_test.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) {
+	sent := time.Now().Add(-sentAgo).UTC()
+	p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0,
+		Modules: map[string]*inventory.PlanModule{
+			"a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
+			"b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}},
+		}}
+	r := inventory.Reported{Node: "ace", Sent: &sent}
+	if reported {
+		at := sent.Add(time.Minute)
+		r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied
+	}
+	return p, []inventory.Reported{r}
+}
+
+// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR
+// 0406 §5); one that reported on the send, or that is silent already, is none.
+func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) {
+	now := time.Now().UTC()
+	p, reports := rolloutOf(5*time.Minute, false)
+	waits := silentWaitsOf(p, reports, now, nil)
+	if len(waits) != 1 {
+		t.Fatalf("waits = %v; want the one machine", waits)
+	}
+	for w, at := range waits {
+		if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute {
+			t.Errorf("wait %+v sent at %v is wrong", w, at)
+		}
+	}
+	p, reports = rolloutOf(5*time.Minute, true)
+	if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
+		t.Errorf("a machine that reported on its send is waited on: %v", waits)
+	}
+	p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false)
+	if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
+		t.Errorf("a machine silent past its time is waited on again: %v", waits)
+	}
+	p, reports = rolloutOf(time.Minute, false)
+	p.State = inventory.PlanDone
+	if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 {
+		t.Errorf("a plan that ended is waiting: %v", waits)
+	}
+}
+
+// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait.
+func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) {
+	since := time.Now().Add(-time.Minute).UTC()
+	g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since}
+	p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}}
+	waits := silentWaitsOf(p, nil, time.Now(), nil)
+	if len(waits) != 2 {
+		t.Fatalf("waits = %v; want g14 and novox", waits)
+	}
+	g.Verdict = inventory.GatePassed
+	if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 {
+		t.Errorf("a gate with its verdict still waits: %v", waits)
+	}
+}
+
+// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted.
+func TestTheKeysFollowTheWaits(t *testing.T) {
+	t1 := time.Now().UTC()
+	t2 := t1.Add(time.Minute)
+	a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"}
+	arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1})
+	if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) {
+		t.Errorf("armed %v; want b (sent again) and c (no key)", arm)
+	}
+	if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) {
+		t.Errorf("dropped %v; want the key with no wait", drop)
+	}
+	if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second {
+		t.Errorf("a wait long over has a key of %v; want the least, one second", ttl)
+	}
+}
+
+func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker {
+	l := newLooker(func(ctx context.Context) bool {
+		looks.Add(1)
+		if block != nil {
+			<-block
+		}
+		return failures.Add(-1) >= 0
+	})
+	l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond
+	return l
+}
+
+func eventually(t *testing.T, what string, ok func() bool) {
+	t.Helper()
+	for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
+		if ok() {
+			return
+		}
+	}
+	t.Fatalf("never: %s", what)
+}
+
+// Asks made while a look runs are not lost and are one look after it, not one each.
+func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) {
+	var failures, looks atomic.Int32
+	block := make(chan struct{})
+	l := lookerCounting(&failures, &looks, block)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	go l.run(ctx)
+	l.ask()
+	eventually(t, "the first look starts", func() bool { return looks.Load() == 1 })
+	for i := 0; i < 20; i++ {
+		l.ask()
+	}
+	block <- struct{}{} // the first look ends
+	eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 })
+	block <- struct{}{}
+	time.Sleep(60 * time.Millisecond)
+	if n := looks.Load(); n != 2 {
+		t.Errorf("%d looks for 21 asks; want 2", n)
+	}
+}
+
+// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails
+// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry).
+func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) {
+	var failures, looks atomic.Int32
+	failures.Store(3) // three looks fail, then one is good
+	l := lookerCounting(&failures, &looks, nil)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	go l.run(ctx)
+	l.ask()
+	eventually(t, "four looks", func() bool { return looks.Load() == 4 })
+	time.Sleep(200 * time.Millisecond)
+	if n := looks.Load(); n != 4 {
+		t.Errorf("%d looks; the good one should have ended the retries at 4", n)
+	}
+	l.mu.Lock()
+	armed, backoff := l.retry != nil, l.backoff
+	l.mu.Unlock()
+	if armed || backoff != 0 {
+		t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff)
+	}
+}
+
+func TestTheRetryWaitDoublesToItsMost(t *testing.T) {
+	l := newLooker(func(context.Context) bool { return true })
+	l.base, l.most = 5*time.Second, time.Minute
+	var seen []time.Duration
+	for i := 0; i < 6; i++ {
+		l.after(true)
+		seen = append(seen, l.backoff)
+		l.retry.Stop()
+		l.retry = nil
+	}
+	want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute}
+	for i := range want {
+		if seen[i] != want[i] {
+			t.Fatalf("waits %v; want %v", seen, want)
+		}
+	}
+}
+
+// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known.
+func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) {
+	var failures, looks atomic.Int32
+	l := lookerCounting(&failures, &looks, nil)
+	lookerMu.Lock()
+	theLooker = l
+	lookerMu.Unlock()
+	t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() })
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	go l.run(ctx)
+
+	planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone})
+	t.Cleanup(func() {
+		walkEnds.mu.Lock()
+		defer walkEnds.mu.Unlock()
+		if t := walkEnds.timers["walk-1"]; t != nil {
+			t.Stop()
+		}
+		delete(walkEnds.timers, "walk-1")
+	})
+	eventually(t, "a look", func() bool { return looks.Load() == 1 })
+	walkEnds.mu.Lock()
+	_, armed := walkEnds.timers["walk-1"]
+	walkEnds.mu.Unlock()
+	if !armed {
+		t.Error("a walk saved done is not armed to be recorded")
+	}
+	askToLook() // no panic, no looker, once it is taken away
+}
+
+// The tier-sends a step made are said once each, per machine, with the modules it was sent.
+func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) {
+	sends := &sendsMade{}
+	ctx := withSendsMade(context.Background(), sends)
+	at := time.Now().UTC()
+	noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at)
+	noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at)
+	noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept
+	got := sentBodies("plan-9", sends)
+	if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" ||
+		got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" {
+		t.Errorf("sent bodies = %+v", got)
+	}
+}
+
+// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5).
+func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) {
+	js, err := broker.Dial(testbus.URL(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(js.Close)
+	if err := js.EnsureControllerBuckets(); err != nil {
+		t.Fatal(err)
+	}
+	var failures, looks atomic.Int32
+	l := lookerCounting(&failures, &looks, nil)
+	lookerMu.Lock()
+	theLooker = l
+	lookerMu.Unlock()
+	t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) })
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	go l.run(ctx)
+	if err := startRolloutSilent(ctx, js); err != nil {
+		t.Fatal(err)
+	}
+	c := silentClockOf()
+	w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"}
+	if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Reported(ctx, w); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(1800 * time.Millisecond)
+	if n := looks.Load(); n != 0 {
+		t.Fatalf("a key a report deleted asked for %d looks", n)
+	}
+	if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 })
+}

cmd/mesh-controller/push.go

@@ -180,9 +180,15 @@ func serve(ctx context.Context) (err error) {
 	// And build results nobody was waiting for. A build triggered any other way than `build`
 	// would otherwise be reported into the void, which is the same as not reporting it.
 	server.Records(builds{inv, open})
-	// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
-	// machines to report moves when they have, and a plan left by a replaced controller resumes.
-	go planTicker(ctx, open)
+	// Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the
+	// tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left
+	// by a replaced controller resumes from the first look, which this asks for. No timer looks every so often.
+	// The wait for each machine's report is a key on the bus that expires; where the bus will not take it a timer of its
+	// own for that wait looks instead.
+	if err := startRolloutSilent(ctx, server.JetStream()); err != nil {
+		fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err)
+	}
+	startPlanLooks(ctx, open)
 	// And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes,
 	// which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it;
 	// nothing polls (issue 973).
@@ -254,6 +260,9 @@ func serve(ctx context.Context) (err error) {
 	// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
 	inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
 		sayPlanMoved(ctx, bus, open.inventory, p)
+		// And the plan's next step is looked at (ADR 0406 §5): a build, a send kept or a command's move is followed
+		// by the look that asks the next tier, in place of the timer that did.
+		planKept(p)
 		// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
 		if p.WalkSaid == inventory.WalkEnded {
 			go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)

cmd/mesh-controller/release_plan.go

@@ -24,9 +24,9 @@ import (
 //
 // The handler that hears the merge computes the plan from the catalogue's one dependency relation,
 // writes it to the store, asks the first tier and returns — the receive loop is never held by a
-// build. Every outcome taken in advances the plan it belongs to; a ticker advances what outcomes
-// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
-// plan where it left it.
+// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can
+// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report);
+// a controller replaced mid-plan finds the plan where it left it and looks first.
 
 // gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
 // novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
@@ -668,29 +668,6 @@ func askedAny(p inventory.Plan) bool {
 	return false
 }
 
-// advancePlans moves every open plan as far as the facts allow: a tier whose modules are all built
-// and whose gates are applied gives way to the next; the last tier done is the plan done. Called
-// after every outcome and on a timer, so a plan waiting on a machine's report moves when it comes.
-//
-// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two
-// controllers — the old and the new while a machine hands its controller over — would each ask a
-// tier the other had just asked. Taken without waiting: whoever holds the plans is moving them.
-func advancePlans(ctx context.Context, open *stores) {
-	release, err := open.inventory.HoldPlans(ctx, false)
-	if err != nil {
-		if !errors.Is(err, inventory.ErrPlansBusy) {
-			fmt.Printf("plans: cannot hold them: %v\n", err)
-		}
-		return
-	}
-	defer release()
-	// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
-	if _, err := releaseBacklog(ctx, open, ""); err != nil {
-		fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
-	}
-	advanceHeld(ctx, open)
-}
-
 // keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold
 // on the plans so it never lengthens it: measured, never acted on, and an error only said.
 func keepWalkPhases(ctx context.Context, open *stores) {
@@ -699,8 +676,13 @@ func keepWalkPhases(ctx context.Context, open *stores) {
 	}
 }
 
-// advanceHeld is advancePlans for a caller already holding the plans.
-func advanceHeld(ctx context.Context, open *stores) {
+// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules
+// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says
+// whether a step failed, which the look at the plans (lookAtPlans) retries.
+//
+// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the
+// old and the new while a machine hands its controller over — would each ask a tier the other had just asked.
+func advanceHeld(ctx context.Context, open *stores) (failed bool) {
 	inv := open.inventory
 	// A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407).
 	defer func() {
@@ -711,15 +693,15 @@ func advanceHeld(ctx context.Context, open *stores) {
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
 		fmt.Printf("plans: cannot read them: %v\n", err)
-		return
+		return true
 	}
 	if len(plans) == 0 {
-		return
+		return false
 	}
 	edges, err := inv.Dependencies(ctx)
 	if err != nil {
 		fmt.Printf("plans: cannot read the dependencies: %v\n", err)
-		return
+		return true
 	}
 	rollsOut := func(module string) bool {
 		u, err := inv.UpgradeOf(ctx, module)
@@ -733,10 +715,12 @@ func advanceHeld(ctx context.Context, open *stores) {
 			// stood still for half an hour was written every few seconds, its revision moved, `plan-moved`
 			// said on the bus, and its last save read as when it began building.
 			before := planSnapshot(*p)
-			moved, err := advanceOnce(ctx, open, p, edges, rollsOut)
+			sends := &sendsMade{}
+			moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut)
 			if err != nil {
+				failed = true
 				// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
-				// the state is left as it was and the step is tried again on the next tick. Said and
+				// the state is left as it was and the step is tried again by the next look. Said and
 				// kept when it is new: the same refusal on every tick is one fact, not one per tick.
 				p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
 				// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
@@ -757,14 +741,18 @@ func advanceHeld(ctx context.Context, open *stores) {
 			if planSnapshot(*p) != before {
 				if err := inv.SavePlan(ctx, p); err != nil {
 					fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
+					failed = true
 					break
 				}
+				// Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent.
+				saySends(ctx, *p, sends)
 			}
 			if !moved {
 				break
 			}
 		}
 	}
+	return failed
 }
 
 // planSnapshot is a plan as a save would write it, to compare before and after a step: what the store
@@ -1066,6 +1054,16 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			p.Modules[m].SentAt = &now
 			p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat)
 		}
+		// One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974).
+		for _, n := range sent {
+			var carried []string
+			for _, m := range rest {
+				if slices.Contains(restTo[m], n) {
+					carried = append(carried, m)
+				}
+			}
+			noteSends(ctx, p.Tier, []string{n}, carried, now)
+		}
 		fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
 			strings.Join(sent, ", "))
 		return true, nil
@@ -1338,6 +1336,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 			s.GatedBy = lead
 		}
 	}
+	noteSends(ctx, p.Tier, sent, modules, now)
 	p.State = inventory.PlanRolling
 	p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
 		strings.Join(sent, ", "))
@@ -1652,23 +1651,6 @@ func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) {
 	p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `push --behind` sends them"
 }
 
-// planTicker advances open plans on a timer, for the steps outcomes alone cannot take.
-func planTicker(ctx context.Context, open *stores) {
-	advancePlans(ctx, open)
-	keepWalkPhases(ctx, open)
-	tick := time.NewTicker(30 * time.Second)
-	defer tick.Stop()
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-tick.C:
-			advancePlans(ctx, open)
-			keepWalkPhases(ctx, open)
-		}
-	}
-}
-
 // planLine is one plan as `status` says it, given the least bound of a tier.
 func planLine(p inventory.Plan, now time.Time) string {
 	return planLineWith(p, now, pauseView{}, tierAtLeast)

cmd/mesh-controller/rollout_silent_bus.go

@@ -49,8 +49,10 @@ func parseSilentKey(key string) (silentWait, bool) {
 
 // silentClock is the bus's side of the wait for a machine's report.
 type silentClock interface {
-	// Arm puts a wait's key, to live ttl: made again if the machine is sent again.
-	Arm(ctx context.Context, w silentWait, ttl time.Duration) error
+	// Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was.
+	Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error
+	// Held is every wait that has a key now, with when its send was.
+	Held(ctx context.Context) (map[silentWait]time.Time, error)
 	// Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller.
 	Reported(ctx context.Context, w silentWait) error
 }
@@ -94,13 +96,13 @@ func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(sile
 	return &busSilent{kv: kv}, nil
 }
 
-func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration) error {
+func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
 	key := silentKey(w)
 	// Made again, not changed: a key's time to live is set when it is made.
 	if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
 		return err
 	}
-	_, err := s.kv.Create(ctx, key, []byte(time.Now().UTC().Format(time.RFC3339)), jetstream.KeyTTL(leastSecond(ttl)))
+	_, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl)))
 	return err
 }
 
@@ -119,3 +121,30 @@ func leastSecond(d time.Duration) time.Duration {
 	}
 	return (d + time.Second - 1) / time.Second * time.Second
 }
+
+func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) {
+	keys, err := s.kv.Keys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	held := map[silentWait]time.Time{}
+	for _, key := range keys {
+		w, ok := parseSilentKey(key)
+		if !ok {
+			continue
+		}
+		e, err := s.kv.Get(ctx, key)
+		if err != nil {
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				continue
+			}
+			return nil, err
+		}
+		at, _ := time.Parse(time.RFC3339Nano, string(e.Value()))
+		held[w] = at
+	}
+	return held, nil
+}

cmd/mesh-controller/rollout_silent_bus_test.go

@@ -57,7 +57,7 @@ func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (s
 func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
 	s, heard := silentOnABus(t)
 	want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
-	if err := s.Arm(t.Context(), want, time.Second); err != nil {
+	if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
 		t.Fatal(err)
 	}
 	if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want {
@@ -70,7 +70,7 @@ func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) {
 	s, heard := silentOnABus(t)
 	ctx := t.Context()
 	w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"}
-	if err := s.Arm(ctx, w, time.Second); err != nil {
+	if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil {
 		t.Fatal(err)
 	}
 	if err := s.Reported(ctx, w); err != nil {
@@ -90,11 +90,11 @@ func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) {
 	s, heard := silentOnABus(t)
 	ctx := t.Context()
 	w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"}
-	if err := s.Arm(ctx, w, 2*time.Second); err != nil {
+	if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
 		t.Fatal(err)
 	}
 	time.Sleep(1200 * time.Millisecond)
-	if err := s.Arm(ctx, w, 2*time.Second); err != nil {
+	if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil {
 		t.Fatal(err)
 	}
 	if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok {

cmd/mesh-controller/status_summary.go

@@ -210,6 +210,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
 	if news {
 		l.summary.nudge()
 	}
+	// A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the
+	// machine, and the look is what deletes its key, knowing the report was on that send.
+	if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil {
+		askToLook()
+	}
 	// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
 	// raised or cleared from it. Only from an account of the machine the store took.
 	if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&

internal/broker/nats.go

@@ -378,10 +378,15 @@ var DegradableGrants = []DegradableGrant{
 		Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " +
 			"window setting says (ADR 0407 decision 1, issue 973)",
 	},
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".event.sent",
+		Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
+			"be said is printed and goes on (ADR 0406 §5, issue 974)",
+	},
 	{
 		Subject: "$KV." + RolloutSilentBucket + ".>",
-		Reason: "refused: the controller advances a rollout on the machines' reports alone, and a machine never heard " +
-			"from is found when the plan is next looked at (ADR 0406 §5, issue 974)",
+		Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " +
+			"for that wait (ADR 0406 §5, issue 974)",
 	},
 }
 

internal/broker/states_agreement_test.go

@@ -34,6 +34,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
 	states = append(states, link.KeyChecked)
 	// And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
 	states = append(states, link.KeyPlanMoved)
+	// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
+	states = append(states, link.KeySent)
 	// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
 	states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged)
 	for _, event := range states {

internal/broker/streams.go

@@ -368,6 +368,8 @@ var ControllerStates = []string{"applied", "refused", "built-before",
 	"checked",
 	// And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from.
 	"plan-moved",
+	// And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
+	"sent",
 	// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
 	// each registered module's declared config (design 57 F3).
 	"assignment-changed", "declared-config-changed", "predefined-changed"}

internal/broker/testdata/composed.conf

@@ -24,7 +24,7 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/catalogue/seats.go

@@ -108,6 +108,8 @@ var defaultSeats = append([]Seat{
 			"checked",
 			// A walk kept in a new state, for the delivery it walks (novox/hq ADR 0239).
 			"plan-moved",
+			// A tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974).
+			"sent",
 			// A key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3).
 			"assignment-changed", "declared-config-changed", "predefined-changed"},
 		Serves: ControllerVerbs},

internal/link/events.go

@@ -76,6 +76,10 @@ const (
 	// KeyChecked: a pull request's merge check was judged (novox/hq to-be 45 §9) — the verdict, its
 	// summary and its report, for the forge's holder to set as the pull request's status.
 	KeyChecked = "checked"
+	// KeySent: a plan's tier was sent to a machine, and the send kept (novox/hq ADR 0406 §5, issue 974): the plan, the tier,
+	// the machine, the modules and when. What an outsider hears of a rollout; the controller's own next step does not
+	// wait for it.
+	KeySent = "sent"
 	// KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines —
 	// was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its
 	// steps from. Said after every save of the serving controller; a delivery's holder also asks for the