Ask the operator for a condition's answers and act on the warrant, so release, stop, start and restart can be answered from any channel that proves who answered (hq ADR 0259)

This commit is contained in:
jochen
2026-10-08 18:32:13 +02:00
parent a1b7be8896
commit 7ea2ba4ea7
27 changed files with 1709 additions and 31 deletions
+53
View File
@@ -0,0 +1,53 @@
package broker
import (
"slices"
"testing"
)
// novox/hq ADR 0259 §6: the controller asks the operator through the router's seat as any user of it, under
// its own name, hears its own warrants, reads its own record, and calls the verbs a warrant chooses.
func TestTheControllerAsksUnderItsOwnNameAndCallsTheVerbsAWarrantChooses(t *testing.T) {
records := Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{"anchor": {
{Module: "messenger", Holds: []Seat{operatorChannel()}},
}}}
users, err := Users(records)
if err != nil {
t.Fatal(err)
}
got := perms(t, users[0])
for _, s := range []string{
"mesh.seat.operator-channel.accept.ask.mesh-controller",
"mesh.seat.operator-channel.accept.cancel.mesh-controller",
"$JS.API.DIRECT.GET.KV_messenger_asks.$KV.messenger_asks.mesh-controller.c1",
"mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stop",
"mesh.seat.node-service-manager.tool.restart.g14", "mesh.seat.mesh-controller.tool.plans",
} {
if !allowed(got.Publish, s) {
t.Errorf("the controller may not publish %s", s)
}
}
for _, s := range []string{
"mesh.seat.operator-channel.accept.ask.mesh-delivery",
"mesh.seat.operator-channel.event.decided.mesh-controller",
// (A direct get of another asker's record is not refused here: the controller holds the whole
// JetStream API, as the only writer of stream definitions.)
"mesh.seat.node-service-manager.tool.stop.g14",
} {
if allowed(got.Publish, s) {
t.Errorf("the controller may publish %s", s)
}
}
if !allowed(got.Subscribe, DecidedSubject) || allowed(got.Subscribe, "mesh.seat.operator-channel.event.decided.mesh-delivery") {
t.Error("the controller does not hear exactly its own warrants")
}
// Its events consumer carries them, so a controller that was away hears what was decided meanwhile.
if !slices.Contains(ControllerFollows, DecidedSubject) {
t.Error("the controller does not follow its warrants")
}
// Without a holder of the seat it is granted no ask at all.
alone, _ := Users(Records{Nodes: []string{"anchor"}, Assigned: map[string][]Declared{}})
if allowed(perms(t, alone[0]).Publish, "mesh.seat.operator-channel.accept.ask.mesh-controller") {
t.Error("asked a seat nobody holds")
}
}
+21 -2
View File
@@ -34,8 +34,15 @@ var (
// LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance
// allowed to act writes by compare-and-set and renews; its revision when taken is the epoch.
LeaseBucket = BucketName(ControllerSeat, "lease")
// AskedBucket keeps what the controller asked the operator about its conditions (novox/hq ADR 0259):
// each ask by its id, its options and the actions they stand for, how it ended and whether the
// controller acted on its warrant — so a restart neither asks twice nor acts twice.
AskedBucket = BucketName(ControllerSeat, "asked")
)
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
const AskedKeptFor = 30 * 24 * time.Hour
// LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed
// every five. The bucket's age, so the bus forgets a holder that stopped renewing.
const LeaseTTL = 15 * time.Second
@@ -59,12 +66,12 @@ const (
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
func IsControllerBucket(bucket string) bool {
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
bucket == ConditionHistoryBucket || bucket == LeaseBucket
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -149,6 +156,18 @@ func (j *JetStream) EnsureControllerBuckets() error {
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", ConditionHistoryBucket, err)
}
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: AskedBucket,
Description: "what the controller asked the operator about its conditions, and what came of each (novox/hq " +
"ADR 0259): written by the controller alone; an ask acted on is acted on once",
History: 1,
TTL: AskedKeptFor,
MaxValueSize: 32 << 10,
MaxBytes: 32 << 20,
Storage: jetstream.FileStorage,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err)
}
return nil
}
+24 -1
View File
@@ -186,8 +186,17 @@ var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}}
// VerbsTheControllerAsksTheDeliveryOwner are the mesh-delivery seat's verbs the controller calls (novox/hq
// ADR 0239): its self-check reads `stalled`, and healer H2 takes the one transition the table allows
// through `close`. A mesh seat's verb is flat: no machine in the subject.
//
// And, since novox/hq ADR 0259, `release` and `stop`: the controller asks the operator for them about a
// delivery held past its bound, and calls them on the operator's warrant, with its why.
var VerbsTheControllerAsksTheDeliveryOwner = []SeatVerb{{Seat: "mesh-delivery", Verb: "stalled"},
{Seat: "mesh-delivery", Verb: "close"}}
{Seat: "mesh-delivery", Verb: "close"}, {Seat: "mesh-delivery", Verb: "release"}, {Seat: "mesh-delivery", Verb: "stop"}}
// VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
// chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat.
var VerbsTheControllerActsOnAWarrant = []SeatVerb{{Seat: "node-service-manager", Verb: "restart"},
{Seat: ControllerSeat, Verb: "plans"}}
// perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the
// holder's machine (novox/hq ADR 0219): `paused.<node>`, the build agent saying whether it takes work.
@@ -387,6 +396,20 @@ func PermissionsFor(p Principal) (Permissions, error) {
for _, v := range VerbsTheControllerAsksTheDeliveryOwner {
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb)
}
// And the verbs a warrant chooses (novox/hq ADR 0259): a node seat's on any machine, its own flat.
for _, v := range VerbsTheControllerActsOnAWarrant {
if v.Seat == ControllerSeat {
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb)
continue
}
pub = append(pub, "mesh.seat."+v.Seat+".tool."+v.Verb+".*")
}
// And asking the operator (novox/hq ADR 0259): an ask and its cancel under its own name, its warrants
// heard under its own name, the record of its asks read under its own name — as any user of the seat,
// derived the same way, from the seat its holder declares.
tp, ts := SeatTrafficOf(ControllerSeat, nil, p.Uses, nil).grants()
pub = append(pub, tp...)
sub = append(sub, ts...)
// And asks who answers (novox/hq to-be 45 §4, D3): the self-check finds every seat's holder by
// the same discovery the console reads. The question only; the answers come to its own inbox.
pub = append(pub, "$SRV.INFO")
+11
View File
@@ -270,8 +270,19 @@ var ControllerFollows = []string{
// seat to check before it merges — every machine of the facts snapshot composed with the change.
// Appended, because the index is a name.
moduleEventSubject("gitea", "pull.updated"),
// **The operator's answers to what the controller asked** (novox/hq ADR 0259): the router's warrant, or
// the end of an ask without one, said to the controller alone under its own name. On the stream, so a
// controller that was away hears what was decided meanwhile. Appended, because the index is a name.
DecidedSubject,
}
// AsksSeat is the seat an ask is made on and its warrant heard from (novox/hq ADR 0259): the router's.
const AsksSeat = "operator-channel"
// DecidedSubject is where the router says the controller's warrants: the seat's event named by the
// controller as its caller.
var DecidedSubject = seatEventSubject(AsksSeat, "decided."+ControllerSeat)
// The provider standing events, by their local names. Written here as well as in the catalogue
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
const (
+2 -2
View File
@@ -24,8 +24,8 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-service-manager.tool.restart.*"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
+19 -1
View File
@@ -78,7 +78,7 @@ type Records struct {
// is a mesh that cannot be told anything, and there is no state of the records in which that is
// correct.
func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController}}
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r)}}
for _, node := range sortedCopy(r.Nodes) {
witness := false
@@ -195,3 +195,21 @@ func sortedNames(in map[string][]string) []string {
// controllerModule is the controller's module: the machine assigned it witnesses its upgrades.
const controllerModule = "mesh-controller"
// asksSeatOf is the seat an ask is made on, as its holder declares it (novox/hq ADR 0259): the controller
// asks the operator through it like any other user, and is granted what its declaration names for a caller.
// None while nothing holds it.
func asksSeatOf(r Records) []Seat {
for _, node := range sortedCopy(r.Nodes) {
for _, d := range r.Assigned[node] {
for _, s := range d.Holds {
if s.Name == AsksSeat && namesVerb(s.ByCaller, "ask") {
seat := s
seat.Kind, seat.Capabilities = "", nil
return []Seat{seat}
}
}
}
}
return nil
}
+12 -1
View File
@@ -53,8 +53,19 @@ type Action struct {
Verb string `json:"verb"`
Machine string `json:"machine,omitempty"`
Arguments map[string]string `json:"arguments,omitempty"`
// Level is how much proof its answer needs (novox/hq ADR 0234 §8, ADR 0259): LevelAcknowledge for what
// any granted principal may already do, LevelApprove for what only the operator's proven word does.
// The controller asks for every action, and performs the one chosen on the warrant the router issues.
Level string `json:"level,omitempty"`
}
// The assurance levels an action's answer needs (novox/hq ADR 0234 §8): acknowledge, approve. Destroy is
// not asked for by any condition: nothing carries its second proof yet.
const (
LevelAcknowledge = "acknowledge"
LevelApprove = "approve"
)
// The two verdicts an explanation opens with.
const (
NothingToDo = "Nothing for you to do."
@@ -66,7 +77,7 @@ const (
// only kind of answer a desk click performs until answers are authorised (novox/hq ADR 0258). Its cause
// marks it as an answer, which the hand-act log does not count as a repair.
func SilenceAction(key string) Action {
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions",
return Action{Label: "Silence for a week", Verb: "mesh-controller.conditions", Level: LevelAcknowledge,
Arguments: map[string]string{"silence": key, "for": "7d", "why": "", "cause": CauseOperatorAnswer}}
}
+4
View File
@@ -47,6 +47,10 @@ var Contracts = map[string]Contract{
KindPullUpdated: {Unordered: "a pull request's head, asked to be checked: each head is its own commit, and its " +
"verdict is set on that commit alone, so a head heard late is checked and judged as itself and never " +
"stands for a newer one (novox/hq to-be 45 §9)"},
KindDecided: {Unordered: "the router's word on one ask, by the ask's id: an ask is ended once, by compare-and-set " +
"at the router, and the controller acts on it once, recording that it did under the ask's id — so a word " +
"heard again, or late, does nothing more (novox/hq ADR 0259)",
Tests: []string{"TestAWarrantIsActedOnOnce"}},
KindCatchUp: {Unordered: "a catalogue asking what it missed: answered from the record, whenever asked"},
KindProvisioner: {Unordered: "a provider's newest word about a consumer, said again every fifteen minutes " +
"while it holds (ADR 0224): the condition keeps the last observed, and S8 says when the words stop. " +
+10
View File
@@ -49,6 +49,16 @@ type HandAct struct {
Kind string `json:"kind,omitempty"`
// Carried is what such a push moved, one "module from → to" per module, so the log says it.
Carried []string `json:"carried,omitempty"`
// Via, Ask, Proofs and RequestedBy are an act the operator chose on a warrant (novox/hq ADR 0234 §8, ADR
// 0259): the channel it came through (module and kind, and how the sender was known), the ask's id, the
// proofs present (P1, P2, P3), and what asked (a condition's key). By then names the operator as that
// kind's identity. Absent from every other act.
Via string `json:"via,omitempty"`
Ask string `json:"ask,omitempty"`
Proofs []string `json:"proofs,omitempty"`
RequestedBy string `json:"requested-by,omitempty"`
// Outcome is what came of an act recorded after it was done: done, or the verb's refusal.
Outcome string `json:"outcome,omitempty"`
}
// KindRecordedBuilds is a push that only moved recorded builds: the person's word their `record` policy
+3
View File
@@ -44,6 +44,9 @@ const (
// KindPullUpdated is the forge announcing a pull request's new head: checked before it merges
// (novox/hq to-be 45 §9).
KindPullUpdated = "pull-updated"
// KindDecided is the router's warrant for an ask the controller made, or that ask's end without one
// (novox/hq ADR 0259): said to the controller alone, under its own name.
KindDecided = "decided"
)
// Control is one thing a node or a module said, as the controller must act on it.
+3 -1
View File
@@ -53,7 +53,7 @@ func Nats(js *broker.JetStream) Inbound {
// whatever was asked for — and not at all when nothing was.
func (n *natsInbound) Also(kind string) error {
switch kind {
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated:
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated, KindDecided:
n.follows[kind] = true
return nil
default:
@@ -259,6 +259,8 @@ func kindOfSubject(subject string) (string, bool) {
return KindSourceMoved, true
case PullUpdatedSubject:
return KindPullUpdated, true
case broker.DecidedSubject:
return KindDecided, true
case BuildOutcome(), BuildOutcomeOf(TheBuildMachineBefore):
// A build's outcome is the role's event now, so it arrives on the events stream rather than
// the control branch — and is acted on by the same handler, because what the controller does
+38 -1
View File
@@ -70,7 +70,7 @@ type Checker interface {
}
// PullUpdatedSubject is where the forge's pull requests land: the controller's own follow of them.
var PullUpdatedSubject = broker.ControllerFollows[len(broker.ControllerFollows)-1]
var PullUpdatedSubject = "mesh.mod.gitea.event.pull.updated"
// Server acts on what nodes and modules say.
//
@@ -96,6 +96,8 @@ type Server struct {
checker Checker
// healths keeps what machines say of their long-running resources (novox/hq ADR 0240).
healths Healths
// decider acts on the operator's warrants for what the controller asked (novox/hq ADR 0259).
decider Decider
log *log.Logger
// giveUp is how long one message is held for the store; zero means GiveUpAfter.
@@ -138,6 +140,21 @@ func (s *Server) Checks(c Checker) error {
return nil
}
// Decider is what the controller does with the router's word on an ask it made (novox/hq ADR 0259): act
// on a warrant once, or record how the ask ended without one.
type Decider interface {
Decided(ctx context.Context, body []byte) error
}
// Decides says what to do about the router's word on the controller's asks, and asks for it delivered.
func (s *Server) Decides(d Decider) error {
if err := s.inbound.Also(KindDecided); err != nil {
return err
}
s.decider = d
return nil
}
// Answers says what to do about a catalogue's catch-up request, and asks for them to be delivered.
func (s *Server) Answers(r Replayer) error {
if err := s.inbound.Also(KindCatchUp); err != nil {
@@ -210,6 +227,8 @@ func (s *Server) act(ctx context.Context, m Control) {
s.provisioner(ctx, m)
case KindPullUpdated:
s.pullUpdated(ctx, m)
case KindDecided:
s.decided(ctx, m)
default:
// Dropped: a message nothing understands will not be understood on the next attempt
// either, and asking for it again would spin.
@@ -656,6 +675,24 @@ func (s *Server) pullUpdated(ctx context.Context, m Control) {
_ = m.Took()
}
// decided hands the router's word on an ask to the decider; a failure to keep what it did is held for the
// store, like any word that must not be lost.
func (s *Server) decided(ctx context.Context, m Control) {
if s.decider == nil {
_ = m.Took()
return
}
err := s.decider.Decided(ctx, m.Body())
switch s.decide(ctx, m, "the operator's word on an ask", "", "", err) {
case Hold:
return
}
if err != nil {
s.log.Printf("the operator's word on an ask could not be kept: %v", err)
}
_ = m.Took()
}
// saysWhatItDid states what a machine now runs, or what it would not take, as a fact on the bus
// (novox/hq ADR 0134).
//