Say a walk that waits too long, and a delivery's own stalls (hq ADR 0239)

A walk mesh-delivery never lets go waited for ever with nothing open: S16
says it at 30 minutes, urgent at 4 hours, naming plans go. Phase B: probe
D14 reads the delivery owner's stalled and raises delivery.<id>.stalled,
and H2 takes the table's transition through its close.

internal/broker/nats.go

@@ -158,6 +158,12 @@ var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb:
 // the controller's grant that acts, and only through the step a person starts.
 var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}}
 
+// VerbsTheControllerAsksTheDeliveryOwner are the mesh-delivery seat's verbs the controller calls (novox/hq
+// ADR 0239): its self-check reads `stalled`, and healer H2 takes the one transition the table allows
+// through `close`. A mesh seat's verb is flat: no machine in the subject.
+var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery", Verb: "stalled"},
+	{Seat: "mesh-delivery", Verb: "close"}}
+
 // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
 // holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
 var perMachineEvents = map[string]bool{"paused.*": true}
@@ -352,6 +358,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
 		for _, v := range VerbsTheBusStepAsks {
 			pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
 		}
+		// And the delivery's owner, a mesh seat, asked on its flat subjects (ADR 0239).
+		for _, v := range VerbsTheControllerAsksTheDeliveryOwner {
+			pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb)
+		}
 		// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
 		// the same discovery the console reads. The question only; the answers come to its own inbox.
 		pub = append(pub, "$SRV.INFO")

internal/broker/testdata/composed.conf

@@ -24,7 +24,7 @@ accounts {
     jetstream: enabled
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/conditions/condition.go

@@ -44,11 +44,13 @@ const (
 	ScopeCore     = "core"
 	ScopeProbe    = "probe"
 	ScopeMesh     = "mesh"
+	// ScopeDelivery is a delivery mesh-delivery owns, by its id (novox/hq ADR 0239).
+	ScopeDelivery = "delivery"
 )
 
 // Scopes is every scope, in the order a person reads them.
 var Scopes = []string{ScopeMachine, ScopePlan, ScopeCall, ScopeBuild, ScopeMerge, ScopeProvider,
-	ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh}
+	ScopeSeat, ScopeBus, ScopeCore, ScopeProbe, ScopeMesh, ScopeDelivery}
 
 // Who resolves a condition.
 const (

internal/link/queue.go

@@ -486,3 +486,50 @@ func PausedSaid(js *broker.JetStream, seat string, nodes []string) (map[string]H
 	}
 	return out, nil
 }
+
+// AskMeshSeatTool asks the holder of a mesh-scoped seat one of its verbs, on the seat's flat subject
+// (design 33 §4: a mesh seat's verb carries no machine), and reads its answer (novox/hq ADR 0239: the
+// delivery's owner's `stalled` and `close`).
+func AskMeshSeatTool(ctx context.Context, conn *nats.Conn, seat, verb string, args any, timeout time.Duration) (Answer, error) {
+	body, err := json.Marshal(args)
+	if err != nil {
+		return Answer{}, err
+	}
+	asking, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	subject := SeatToolSubject(seat, verb)
+	refused, stop := refusalsOf(conn, subject)
+	defer stop()
+	type replied struct {
+		msg *nats.Msg
+		err error
+	}
+	done := make(chan replied, 1)
+	go func() {
+		msg, err := conn.RequestWithContext(asking, subject, body)
+		done <- replied{msg, err}
+	}()
+	var reply *nats.Msg
+	select {
+	case r := <-done:
+		reply, err = r.msg, r.err
+	case why := <-refused:
+		cancel()
+		return Answer{}, fmt.Errorf("the bus refused the controller asking %s.%s — its grants do not name %s: %v",
+			seat, verb, subject, why)
+	}
+	switch {
+	case errors.Is(err, nats.ErrNoResponders):
+		return Answer{}, fmt.Errorf("%w: nothing answers %s.%s — its holder is not running, or is older than the verb",
+			ErrNothingServes, seat, verb)
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
+		return Answer{}, fmt.Errorf("%s did not answer %s within %s", seat, verb, timeout)
+	case err != nil:
+		return Answer{}, err
+	}
+	var answer Answer
+	if err := json.Unmarshal(reply.Data, &answer); err != nil {
+		return Answer{}, fmt.Errorf("%s answered %s with something unreadable: %w", seat, verb, err)
+	}
+	return answer, nil
+}