Ask a held move and the builds held after a failed release, and say them on the plan's line (issue 1732, 1320)

cmd/mesh-controller/asker.go

@@ -645,6 +645,10 @@ func doesWords(act conditions.Action) string {
 		return "the delivery is stopped"
 	case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
 		return "the failed builds are asked again"
+	case act.Verb == broker.ControllerSeat+".push" && act.Arguments["node"] != "":
+		return "the held builds are sent to " + act.Arguments["node"]
+	case act.Verb == broker.ControllerSeat+".upgrade" && act.Arguments["release-backlog"] != "":
+		return "the builds held after the failed delivery are released"
 	case strings.HasSuffix(act.Verb, ".restart"):
 		return "its service is restarted on " + act.Machine
 	}

cmd/mesh-controller/held_moves.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
+//
+// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that
+// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and
+// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on
+// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way.
+// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the
+// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends
+// when the machine no longer waits, whoever pushed it.
+//
+// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has
+// its own row (S17).
+
+// kindMovesHeld is S24's kind: machine.<node>.moves-held.
+const kindMovesHeld = "moves-held"
+
+// heldFacts is one machine and the moves of recorded modules that wait there for a person.
+type heldFacts struct {
+	node  string
+	moves []heldMove
+}
+
+// gatherHeldMoves is, per machine, the moves that wait for a person and are no person's own choice, sorted by machine.
+func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) {
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the machines cannot be read: %w", err)
+	}
+	f, err := readMoveFacts(ctx, inv)
+	if err != nil {
+		return nil, err
+	}
+	var out []heldFacts
+	for _, n := range nodes {
+		moves, err := heldMovesOn(ctx, inv, f, n.Name)
+		if err != nil {
+			return nil, err
+		}
+		var asked []heldMove
+		for _, m := range moves {
+			if !m.Chosen {
+				asked = append(asked, m)
+			}
+		}
+		if len(asked) > 0 {
+			out = append(out, heldFacts{node: n.Name, moves: asked})
+		}
+	}
+	return out, nil
+}
+
+// modules are the held moves' modules, in order.
+func (h heldFacts) modules() []string {
+	out := make([]string, 0, len(h.moves))
+	for _, m := range h.moves {
+		out = append(out, m.Module)
+	}
+	return out
+}
+
+// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push
+// names every module whose move waits there as one that may move its data: the operator's Approve is the word
+// ADR 0217 asks for, and the ask says so.
+func (h heldFacts) releaseAction() conditions.Action {
+	return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove,
+		Arguments: map[string]string{"node": h.node, "move": strings.Join(h.modules(), ","), "why": "",
+			"cause": conditions.CauseOperatorAnswer}}
+}
+
+// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does.
+func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) {
+	names := namesWords(h.modules(), 3)
+	headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax)
+	recreates, data := false, false
+	for _, m := range h.moves {
+		switch {
+		case strings.Contains(m.Why, "recreates"):
+			recreates = true
+		default:
+			data = true
+		}
+	}
+	why := "would change what it runs"
+	switch {
+	case recreates && data:
+		why = "would restart a container or touch its data"
+	case recreates:
+		why = "would restart one of its containers"
+	case data:
+		why = "would touch its files or data"
+	}
+	explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
+		"it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+
+		"has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax)
+	needs = "release them " + FromMeshMCPServer
+	resolved = "Resolved: " + names + " is no longer waiting on " + h.node
+	return
+}
+
+// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the
+// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts.
+func watchMovesHeld(f *signalFacts) []conditions.Observation {
+	var out []conditions.Observation
+	for _, h := range f.held {
+		var said []string
+		for _, m := range h.moves {
+			said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why))
+		}
+		headline, explanation, needs, resolved := heldWords(h)
+		out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld,
+			Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
+			Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them",
+				h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node),
+			Said:     strings.Join(said, "; "),
+			Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved,
+			Actions: []conditions.Action{h.releaseAction()}})
+	}
+	return out
+}
+
+// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a
+// person: "module on machine", sorted. The plan's own record says the move waited when it was judged
+// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older
+// than the build now held is not named.
+func planHeld(p inventory.Plan, held []heldFacts) []string {
+	var out []string
+	for _, h := range held {
+		for _, m := range h.moves {
+			s := p.Modules[m.Module]
+			if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) {
+				continue
+			}
+			out = append(out, m.Module+" on "+h.node)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a
+// plan whose module waits for a push is not plainly "done".
+func heldSaid(p inventory.Plan, held []heldFacts) string {
+	waiting := planHeld(p, held)
+	if len(waiting) == 0 {
+		return ""
+	}
+	return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", "))
+}
+
+// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before.
+func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts {
+	held, err := gatherHeldMoves(ctx, inv)
+	if err != nil {
+		return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read
+	}
+	return held
+}

cmd/mesh-controller/held_moves_test.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/asks"
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
+
+// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words.
+func aHeldMailu() heldFacts {
+	return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb",
+		Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}}
+}
+
+func heldConditionOn(f *signalFacts) conditions.Condition {
+	o := watchMovesHeld(f)[0]
+	return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
+		Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
+}
+
+// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with
+// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there.
+func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	f.held = []heldFacts{aHeldMailu()}
+	got := watchMovesHeld(f)
+	if len(got) != 1 {
+		t.Fatalf("%+v", got)
+	}
+	o := got[0]
+	if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning ||
+		o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" {
+		t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key())
+	}
+	if len(o.Actions) != 1 {
+		t.Fatalf("actions: %+v", o.Actions)
+	}
+	a := o.Actions[0]
+	if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove ||
+		a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
+		t.Fatalf("the action is not the push of novox: %+v", a)
+	}
+	if _, takes := a.Arguments["why"]; !takes {
+		t.Fatal("the push takes a why, which the asker fills in with the operator's answer")
+	}
+	for _, want := range []string{"mailu", "novox"} {
+		if !strings.Contains(o.Headline+o.Explanation, want) {
+			t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation)
+		}
+	}
+	if !strings.Contains(o.Said, "recreates 1 of mailu's 12") {
+		t.Errorf("the evidence does not carry the walk's own words: %q", o.Said)
+	}
+	if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax {
+		t.Errorf("headline: %q", o.Headline)
+	}
+}
+
+// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent.
+func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	if got := watchMovesHeld(f); len(got) != 0 {
+		t.Fatalf("%+v", got)
+	}
+}
+
+// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the
+// operator's warrant calls the controller's own push with the machine and the modules, once.
+func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	f.held = []heldFacts{aHeldMailu()}
+	c := heldConditionOn(f)
+	r := newAskerRig(t)
+	r.open = []conditions.Condition{c}
+	if err := r.a.reconcile(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	sent := r.asksSent(t)
+	if len(sent) != 1 || len(sent[0].Options) != 1 {
+		t.Fatalf("asked %+v", sent)
+	}
+	o := sent[0].Options[0]
+	if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" {
+		t.Errorf("the option is %+v", o)
+	}
+	if err := sent[0].Check(r.now); err != nil {
+		t.Errorf("the ask is refused as made: %v", err)
+	}
+	w := r.warrantFor(t, c.Key, "Release")
+	answerWith(t, r, w)
+	answerWith(t, r, w)
+	if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") ||
+		!strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) ||
+		!strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) ||
+		!strings.Contains(r.called[0], "chose Release") {
+		t.Fatalf("called %v", r.called)
+	}
+}
+
+// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them.
+func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) {
+	for _, verb := range []string{"push", "upgrade"} {
+		granted := false
+		for _, v := range broker.VerbsTheControllerActsOnAWarrant {
+			granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb)
+		}
+		if !granted {
+			t.Errorf("the controller may not act on a warrant for %s", verb)
+		}
+	}
+}
+
+func aDoneMailuPlan() inventory.Plan {
+	return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone,
+		Tiers: [][]string{{"a"}, {"b"}, {"mailu"}},
+		Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc",
+			Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}}
+}
+
+// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine
+// no longer holds it, or holds another build of the module, the line says nothing more.
+func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) {
+	p := aDoneMailuPlan()
+	line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()})
+	if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") {
+		t.Errorf("the line: %q", line)
+	}
+	if said := heldSaid(p, nil); said != "" {
+		t.Errorf("a plan whose moves were sent says %q", said)
+	}
+	other := aHeldMailu()
+	other.moves[0].To = "9999999ffff"
+	if said := heldSaid(p, []heldFacts{other}); said != "" {
+		t.Errorf("a plan older than the build now held says %q", said)
+	}
+	p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy}
+	if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" {
+		t.Errorf("a move the plan sent says %q", said)
+	}
+}
+
+// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers
+// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant.
+func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) {
+	saved := backlogNow
+	t.Cleanup(func() { backlogNow = saved })
+	backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person"
+	backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}}
+	got := backlogObservation()
+	if len(got) != 1 || len(got[0].Actions) != 1 {
+		t.Fatalf("raised %+v", got)
+	}
+	a := got[0].Actions[0]
+	if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove ||
+		a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
+		t.Fatalf("the action: %+v", a)
+	}
+	o := got[0]
+	c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
+		Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
+	r := newAskerRig(t)
+	r.open = []conditions.Condition{c}
+	if err := r.a.reconcile(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	sent := r.asksSent(t)
+	if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" {
+		t.Fatalf("asked %+v", sent)
+	}
+	w := r.warrantFor(t, c.Key, "Release")
+	answerWith(t, r, w)
+	answerWith(t, r, w)
+	if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") ||
+		!strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") {
+		t.Fatalf("called %v", r.called)
+	}
+}
+
+// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for
+// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is.
+func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) {
+	saved := backlogNow
+	t.Cleanup(func() { backlogNow = saved })
+	now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC)
+	p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding,
+		Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute),
+		Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}},
+		Note:    refusalNote(0, errors.New("a build waits there for its gate: novox"))}
+	backlogNow.held = ""
+	if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") {
+		t.Errorf("the wait is not said: %q", line)
+	}
+	backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person"
+	line := planLine(p, now)
+	if strings.Contains(line, "LATE") || strings.Contains(line, "building") ||
+		!strings.Contains(line, "until a person releases it") {
+		t.Errorf("a plan held behind a failed release reads %q", line)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -457,6 +457,11 @@ var plainWordings = map[string]func(conditions.Observation) words{
 			Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
 			Resolved:    "Resolved: the delivery is no longer failed"}
 	}),
+	kindMovesHeld: worded(func(o conditions.Observation) words {
+		return words{Headline: "An update waits for you",
+			Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.",
+			Resolved:    "Resolved: the update is no longer waiting"}
+	}),
 	"ask-lost": worded(func(o conditions.Observation) words {
 		return words{Headline: "A build has no result",
 			Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",

cmd/mesh-controller/plain_words_test.go

@@ -260,7 +260,7 @@ func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) {
 	plainExample(t, got[0], "Updates of openrazer and sensors wait for your release",
 		"Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+
 			"sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+
-			"They are not delivered until then, and stay held if you leave them.")
+			"They are not delivered until then, and stay held if you leave them.", "Release")
 }
 
 // **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any

cmd/mesh-controller/plans_follow.go

@@ -153,7 +153,7 @@ func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsk
 	line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
 		on, _ := inv.Running(ctx, module)
 		return on
-	}, waitsOf(ctx, inv, one))
+	}, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv))
 	doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
 	return json.NewEncoder(os.Stdout).Encode(doc)
 }

cmd/mesh-controller/recorded_moves.go

@@ -181,15 +181,24 @@ func planModuleNames(p inventory.Plan) map[string]string {
 	return out
 }
 
-// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
-// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
-// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
-func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
+// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6).
+type heldMove struct {
+	Module   string
+	From, To string
+	// Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold.
+	Why string
+	// Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the
+	// operator is not asked about again: they chose that hold, and the bus has its own condition (S17).
+	Chosen bool
+}
+
+// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order.
+func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) {
 	sent, known, err := inv.SentBuilds(ctx, node)
 	if err != nil || !known {
 		return nil, err
 	}
-	var out []string
+	var out []heldMove
 	for _, m := range sortedKeysOf(sent) {
 		was := sent[m]
 		now, held := f.current[m]
@@ -197,12 +206,25 @@ func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts,
 			continue
 		}
 		if v := f.recordedMove(m, was); !v.rolls {
-			out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said))
+			out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said,
+				Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus})
 		}
 	}
 	return out, nil
 }
 
+// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
+// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
+// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
+func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
+	moves, err := heldMovesOn(ctx, inv, f, node)
+	var out []string
+	for _, h := range moves {
+		out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why))
+	}
+	return out, err
+}
+
 // stringsKeys is a map of lists as a map of its keys, for sortedKeysOf.
 func stringsKeys(m map[string][]string) map[string]string {
 	out := map[string]string{}

cmd/mesh-controller/release.go

@@ -1371,6 +1371,10 @@ func backlogObservation() []conditions.Observation {
 			"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}
 	w := releaseHeldWords(modules, machines)
 	o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
+	// Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own
+	// `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged.
+	o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove,
+		Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}}
 	return []conditions.Observation{o}
 }
 

cmd/mesh-controller/release_plan.go

@@ -2061,6 +2061,12 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
 		// refused ("a build waits there for its gate: <machine> (...)") is kept there. "building" with LATE
 		// hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920).
 		what = p.Note
+		// And when the builds are held after a failed release, they wait for a person, not for a build, and the
+		// operator is asked to release them (novox/hq issue 1732): said so, and not late.
+		if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" {
+			what += " — held after a failed delivery until a person releases it (the operator is asked)"
+			late = ""
+		}
 		return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late)
 	}
 	return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
@@ -2268,7 +2274,7 @@ func plansCommand(ctx context.Context, args []string) error {
 			bounds.of(p.Repository), func(module string) []string {
 				on, _ := inv.Running(ctx, module)
 				return on
-			}, waitsOf(ctx, inv, []inventory.Plan{p})))
+			}, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv)))
 		if r := p.Release; r != nil {
 			// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
 			fmt.Printf("  machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
@@ -2463,11 +2469,12 @@ func plansCommand(ctx context.Context, args []string) error {
 		return on
 	}
 	waits := waitsOf(ctx, inv, plans)
+	held := heldNow(ctx, inv)
 	for _, p := range plans {
 		if p.Batch() {
 			continue
 		}
-		fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits))
+		fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits), heldSaid(p, held))
 	}
 	return nil
 }

cmd/mesh-controller/signals.go

@@ -244,6 +244,14 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
 		}},
+	{Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator",
+		Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)",
+		Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " +
+			"machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " +
+			"pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.",
+		Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3,
+		needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld,
+		newest: func(f *signalFacts) time.Time { return time.Time{} }},
 	{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
 		Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
 		Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +

cmd/mesh-controller/signals_test.go

@@ -171,6 +171,12 @@ var suppressions = map[string]suppression{
 				since: f.now.Add(-time.Second)}}
 		},
 	},
+	// A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732).
+	// Inside: nothing waits there.
+	"S24": {
+		inside: func(f *signalFacts) { f.held = nil },
+		past:   func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} },
+	},
 	// A batch queued behind an open walk past that walk's bound, named.
 	"S19": {
 		inside: func(f *signalFacts) {

cmd/mesh-controller/watchdogs.go

@@ -58,6 +58,9 @@ type signalFacts struct {
 	// failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449).
 	failed    []failedPlanFacts
 	failedErr error
+	// held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732).
+	held    []heldFacts
+	heldErr error
 	// refusedSends are the sends a machine refused within the hour for the generation they were composed
 	// from, each naming its sender (S20, novox/hq issue 234).
 	refusedSends    []inventory.Send
@@ -357,6 +360,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 		f.batches, f.plansErr = gatherBatches(ctx, inv, now)
 	}
 	f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now)
+	f.held, f.heldErr = gatherHeldMoves(ctx, inv)
 	f.loop, f.loopErr = w.gatherLoop()
 	f.asks, f.asksErr = w.gatherAsks(ctx, inv)
 	if open, err := w.keeper.Open(ctx); err != nil {

internal/broker/nats.go

@@ -361,9 +361,11 @@ var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"}
 
 // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
 // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
-// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat.
+// controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release
+// sent by its own `upgrade release-backlog` (novox/hq issue 1732). Named one by one; a node seat's on any machine, a
+// mesh seat's flat.
 var VerbsTheControllerActsOnAWarrant = []SeatVerb{{Seat: "node-service-manager", Verb: "restart"},
-	{Seat: ControllerSeat, Verb: "plans"}}
+	{Seat: ControllerSeat, Verb: "plans"}, {Seat: ControllerSeat, Verb: "push"}, {Seat: ControllerSeat, Verb: "upgrade"}}
 
 // VerbsTheControllerAsksTheOperatorChannel are the operator channel's verbs the controller calls (novox/hq
 // ADR 0380): `tell`, one plain message with no condition behind it — a module's own secret given, named,
@@ -387,6 +389,16 @@ type DegradableGrant struct {
 // works. Every other grant stays required — missing one of those is a controller that comes up and is
 // refused on the first thing it tries, with no fallback that covers it.
 var DegradableGrants = []DegradableGrant{
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".tool.push",
+		Reason: "refused: the ask of a held move (Release) is answered, and the refusal told, while the operator's own " +
+			"push at the terminal still sends the machine (issue 1732; ADR 0259)",
+	},
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".tool.upgrade",
+		Reason: "refused: the ask of the builds held after a failed release is answered, and the refusal told, while the " +
+			"operator's own `upgrade release-backlog` at the terminal still releases them (issue 1732; ADR 0259)",
+	},
 	{
 		Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb,
 		Reason:  "refused: the controller raises the secret-given condition as before (ADR 0380 §9)",

internal/broker/testdata/composed.conf

@@ -29,7 +29,7 @@ accounts {
     ]
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }