novox/mesh-controller
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Gate a release plan's first machine and roll a failed build back there (hq ADR 0236)

Browse Source 
A build that reported applied was sent everywhere; one that then did nothing, served
no tools or broke its machine's word reached every machine. Now the first machine is
judged by the component's health (the core's definitions, as doctor probes H-*, or a
module's own) three times over two minutes within ten; a failing gate puts the previous
build back there once, marks the build, and says it as a condition and an event.
Upgrades roll out by default; the bus is a planned step; a module deleted at its
source is not built (the public-acme plan failure).
This commit is contained in:
jochen
2026-10-06 18:56:54 +02:00
parent 81f497e5dd
commit d9289ef6d4
41 changed files with 3297 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/broker/nats.go
+24
View File
@@ -19,6 +19,8 @@ import (
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/lease"
)
// A Kind is what a principal is, which decides the shape of its authority rather than its
@@ -115,6 +117,12 @@ type Principal struct {
// else — not its declarations, which the node's tool runtime serves for it.
SnapshotsTheBus bool
// WitnessesController is a machine principal whose node-engine witnesses the controller's upgrades:
// the machine runs the controller (novox/hq ADR 0236, lease/witness.go). It may read the lease's one
// key, and nothing else of the bucket, so it can judge a new controller build and put the previous
// one back.
WitnessesController bool
// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
// and never appears here: this file is written to a node's disk and read by a server, and a
// secret that can be read from a configuration file is a secret with a wider blast radius
@@ -432,6 +440,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
// `report` verb healer H1 asks): its own machine's, on core NATS and off any stream. It
// answers through its report, the one thing it already says — no reply to anybody's inbox.
AskReportSubject(p.Node)}
// The node-engine witnesses the core builds it places (novox/hq to-be 45 §8, ADR 0236; the
// contract is lease/witness.go): it asks its own machine's node tools PING, and, where the
// machine runs the controller, reads the lease's one key — read, never written.
pub = append(pub, WitnessSubjects(p)...)
case KindModule:
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
@@ -979,3 +991,15 @@ func announcing(names ...string) []string {
func discovering() []string {
return []string{"$SRV.PING", "$SRV.PING.>", "$SRV.INFO", "$SRV.INFO.>"}
}
// WitnessSubjects are the subjects a machine's node-engine publishes to witness the core builds it
// places (novox/hq ADR 0236, lease/witness.go): its own node tools' PING, and the lease's key where it
// runs the controller. Reads only: a write to the bucket is `$KV.<bucket>.>`, the controller's alone
// (the writers table). The answers come to the machine's own inbox.
func WitnessSubjects(p Principal) []string {
out := []string{lease.PingSubject(p.Node)}
if p.WitnessesController {
out = append(out, lease.LeaseReadSubject(LeaseBucket))
}
return out
}
internal/broker/states_agreement_test.go
+2
View File
@@ -28,6 +28,8 @@ func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) {
states = append(states, link.KeySecretReplaced)
// And every act a healer takes (novox/hq to-be 45 §7).
states = append(states, link.KeyHealerActed)
// And a build put back after its gate failed (novox/hq ADR 0235).
states = append(states, link.KeyRolledBack)
for _, event := range states {
if !slices.Contains(broker.ControllerStates, event) {
t.Errorf("the mesh states %q and its account may not publish it", event)
internal/broker/streams.go
+3 -1
View File
@@ -213,7 +213,9 @@ var ControllerStates = []string{"applied", "refused", "built-before",
"secret-replaced",
// And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh
// made by itself is said like one a person made, never quietly.
"healer-acted"}
"healer-acted",
// And a build put back after its gate failed on its first machine (novox/hq ADR 0235, to-be 45 §8).
"rolled-back"}
// BusAdvisories are what the bus server says about the mesh's own account that the controller
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
internal/broker/testdata/composed.conf
Vendored
+2 -2
View File
@@ -24,7 +24,7 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
@@ -33,7 +33,7 @@ accounts {
subscribe: { allow: ["_INBOX.enrol.one.>"] }
} }
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "mesh.control.one.>"] }
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.INFO.NODES.one", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.report", "mesh.node.one.declare"] }
} }
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
internal/broker/users.go
+10 -1
View File
@@ -72,7 +72,13 @@ func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController}}
for _, node := range sortedCopy(r.Nodes) {
out = append(out, Principal{Kind: KindNode, Node: node})
witness := false
for _, d := range r.Assigned[node] {
if d.Module == controllerModule {
witness = true
}
}
out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness})
// **Where the runtime is assigned, the machine gets one runtime principal in place of the
// runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the
// node: its serving grants are the union of theirs. Every other module keeps its own
@@ -171,3 +177,6 @@ func sortedNames(in map[string][]string) []string {
sort.Strings(out)
return out
}
// controllerModule is the controller's module: the machine assigned it witnesses its upgrades.
const controllerModule = "mesh-controller"
internal/broker/witness_grants_test.go
+37
View File
@@ -0,0 +1,37 @@
package broker
import (
"slices"
"testing"
)
// Every machine's node-engine may ask its own node tools PING; the one running the controller may read
// the lease's key, and nothing else of the bucket (novox/hq ADR 0236).
func TestTheWitnessIsGrantedWhatItReadsAndNoMore(t *testing.T) {
users, err := Users(Records{Nodes: []string{"control", "edge"},
Assigned: map[string][]Declared{"control": {{Module: "mesh-controller"}}}})
if err != nil {
t.Fatal(err)
}
lease := "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder"
for _, u := range users {
if u.Kind != KindNode {
continue
}
perms, err := PermissionsFor(u)
if err != nil {
t.Fatal(err)
}
if !slices.Contains(perms.Publish, "$SRV.PING.node-tools."+u.Node) {
t.Errorf("%s may not ask its node tools: %v", u.Node, perms.Publish)
}
if got := slices.Contains(perms.Publish, lease); got != (u.Node == "control") {
t.Errorf("%s may read the lease: %v", u.Node, got)
}
for _, p := range perms.Publish {
if p == "$KV.mesh-controller_lease.>" || p == "$KV.mesh-controller_lease.holder" {
t.Errorf("%s may write the lease", u.Node)
}
}
}
}