Act under a lease, keep accounts by order, one writer at composition (hq to-be 45 Phase 2)

Two controllers could both act (issue 204), a reconcile's report could
overtake the apply after it and the digest decided (issue 267), and a grant
could make a second writer of a machine's report.

- The lease (internal/lease, ADR 0229): mesh-controller_lease key `holder`,
  15 s age, renewed every 5 s by compare-and-set; the epoch is the revision
  it was taken at. The gate is the clock (stops 3 s before expiry); a refused
  renewal is a loss and the process exits; a holder that stops gives it back.
  serve takes it before asserting the bus. Epochs kept in the store
  (migration 0068 controller_epoch) as a floor: a bucket raised from nothing
  is compacted past it. Unleased (no epoch, S12 urgent) only when nobody
  holds it and the bus will not let it be written. A shell command acts
  under the holder's epoch, or its own lease when none.
- Declarations carry `epoch` inside the signed envelope, only to a machine
  whose latest account carried a report_sequence (mesh-host #35); would-send
  is composed with the epoch last sent. Allot and the send both pass the gate.
- Reports: contract in internal/link/order.go (epoch, sequence,
  report_sequence, older_than, refused_older). Accounts kept by epoch, then
  sequence, then report sequence; older refused, counted; unordered reports
  keep the digest rule. Plans by compare-and-set on a revision, with epoch.
  Conditions and calls carry the epoch and are not written off the lease.
- S12 and S13 (naming the writer by epoch) watched, D5 run; reset of the
  bucket said. Writers table compiled in and enforced in PermissionsFor; the
  controller no longer publishes mesh.control.>. A contract per consumed
  kind, and the empty-on-error lint over the repository.
- mesh-host pinned to its main with the epoch in the validator (D1 validates
  the envelope as sent).

Needs mesh-host's genesis lock with the lease grant (mesh-host PR) for
TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose.
This commit is contained in:
jochen
2026-10-06 12:29:18 +02:00
parent 070ecafc07
commit 2eb9a22c24
59 changed files with 3975 additions and 158 deletions
+33 -2
View File
@@ -31,8 +31,15 @@ var (
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
ConditionsBucket = BucketName(ControllerSeat, "conditions")
ConditionHistoryBucket = BucketName(ControllerSeat, "condition-history")
// LeaseBucket holds the controller's lease (to-be 45 §6): one key, `holder`, which the instance
// allowed to act writes by compare-and-set and renews; its revision when taken is the epoch.
LeaseBucket = BucketName(ControllerSeat, "lease")
)
// LeaseTTL is how long the lease's key lives unrenewed (to-be 45 §6): fifteen seconds, renewed
// every five. The bucket's age, so the bus forgets a holder that stopped renewing.
const LeaseTTL = 15 * time.Second
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
// so the stream holds twice as many messages as it keeps calls.
@@ -52,12 +59,12 @@ const (
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
func IsControllerBucket(bucket string) bool {
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
bucket == ConditionHistoryBucket
bucket == ConditionHistoryBucket || bucket == LeaseBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -74,6 +81,9 @@ func (j *JetStream) EnsureControllerBuckets() error {
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := EnsureLeaseBucket(ctx, js); err != nil {
return err
}
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: CallsBucket,
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
@@ -141,3 +151,24 @@ func (j *JetStream) EnsureControllerBuckets() error {
}
return nil
}
// EnsureLeaseBucket creates the lease's bucket if absent and brings its options to match (to-be 45
// §6). **Before the lease is taken, by any candidate**: it is the lease's own precondition, and asserting
// a bucket that exists changes nothing. File storage, so its revisions — the epochs — outlive a restart of
// the bus; one raised again from nothing is moved past the highest epoch issued when the lease is taken.
func EnsureLeaseBucket(ctx context.Context, js jetstream.JetStream) error {
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: LeaseBucket,
Description: "the controller's lease (novox/hq to-be 45 §6): the key `holder`, written by compare-and-set " +
"by the one controller instance that may act and renewed every five seconds; its revision when taken " +
"is the epoch every declaration and plan write carries",
History: 1,
TTL: LeaseTTL,
MaxValueSize: 4 << 10,
MaxBytes: 1 << 20,
Storage: jetstream.FileStorage,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", LeaseBucket, err)
}
return nil
}
+11 -1
View File
@@ -229,7 +229,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
// stream definitions (design 25 §3), so it alone reaches the JetStream API — and it alone
// issues memberships (novox/hq ADR 0160), which it publishes into the assignments stream
// after each push; refused by the server on 2026-10-01 until this line named them.
pub = []string{"mesh.control.>", "mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
//
// **Not what the machines say** (novox/hq to-be 45 §1): `mesh.control.>` is subscribed, never
// published — a machine's report has one writer, its node-engine, and the controller granted
// that subject would be a second (the writers table refuses it). It was granted from the first
// composition and nothing ever published under it.
pub = []string{"mesh.node.>", "mesh.assignment.>", "$JS.API.>"}
// **And where its consumers deliver.** A push consumer delivers on `_DELIVER.<its name>`,
// and a client bound to it subscribes exactly that; the server refused it for every
// principal the first time one bound a consumer (2026-09-28). Each kind below is granted
@@ -604,6 +609,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
sort.Strings(pub)
sort.Strings(sub)
// One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is
// refused here, at composition, naming the state and its writer.
if err := CheckWriters(p, pub); err != nil {
return Permissions{}, err
}
return Permissions{
Publish: pub,
Subscribe: sub,
+1 -1
View File
@@ -24,7 +24,7 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
allow_responses: { max: 1, ttl: "1m" }
} }
+167
View File
@@ -0,0 +1,167 @@
package broker
import (
"fmt"
"slices"
"strings"
)
// The writers table (novox/hq to-be 45 §1, ADR 0227 rule 1), compiled in.
//
// **Every kind of state the core keeps has one writer; anyone else asks it.** The table is the design's,
// row for row, with what each row writes on the bus where it writes there. Two things read it: the
// composition of every principal's grants (PermissionsFor), which **refuses a grant that lets a
// principal publish on a subject another writes** — so a second writer cannot be granted by accident,
// and the composition says which state and whose — and the test beside it, which walks the rows
// against the design's list and the composition of a whole mesh. Changing a writer is a change to
// this table, through a decision.
// WriterRow is one row of the writers table.
type WriterRow struct {
State string
Writer string
KeptIn string
Others string
// Subjects are the bus subjects a write of this state is a publish to; none for state kept off the
// bus (the controller's store, a module's own) or not built yet.
Subjects []string
// Writes says a principal granted a publish pattern overlapping Subjects is this state's writer —
// given the pattern, because a machine writes its own report and no other's.
Writes func(p Principal, pattern string) bool
// Shared says why more than one principal may publish here; empty for one writer.
Shared string
}
// isController, and the other writers' tests, are who a row's writer is as a principal.
func isController(p Principal, _ string) bool { return p.Kind == KindController }
// ownMachine is a node writing a subject whose third token is its own name, and no wildcard there.
func ownMachine(p Principal, pattern string) bool {
tokens := strings.Split(pattern, ".")
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
}
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
func holdsSeatOf(p Principal, pattern string) bool {
tokens := strings.Split(pattern, ".")
if len(tokens) < 3 {
return false
}
seat := tokens[2]
holds := func(seats []Seat) bool {
return slices.ContainsFunc(seats, func(s Seat) bool { return s.Name == seat })
}
switch p.Kind {
case KindModule:
return holds(p.Holds)
case KindNodeTools:
return slices.ContainsFunc(p.Carries, func(d Declared) bool { return holds(d.Holds) })
}
return false
}
// ownModule is a module publishing under its own name, or the node tools carrying it.
func ownModule(p Principal, pattern string) bool {
tokens := strings.Split(pattern, ".")
if len(tokens) < 3 {
return false
}
module := tokens[2]
switch p.Kind {
case KindModule:
return p.Module == module
case KindNodeTools:
return slices.ContainsFunc(p.Carries, func(d Declared) bool { return d.Module == module })
}
return false
}
// kvOf is a bucket's write subjects.
func kvOf(bucket string) []string { return []string{"$KV." + bucket + ".>"} }
// WritersTable is to-be 45 §1, in its order.
var WritersTable = []WriterRow{
{State: "a machine's declaration", Writer: "controller (lease holder)", KeptIn: "the bus, last per subject",
Others: "read", Subjects: []string{"mesh.node.*.declare"}, Writes: isController},
{State: "a machine's applied state and its report", Writer: "the node-engine's apply queue",
KeptIn: "the machine; the report on the bus", Others: "the reconcile and a delivery enqueue, never apply",
Subjects: []string{"mesh.control.*.report"}, Writes: ownMachine},
{State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket,
Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController},
{State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision",
KeptIn: "the controller's store", Others: "read through plans"},
{State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " +
ConditionHistoryBucket + ")", Others: "raise or clear only through observations the controller reads",
Subjects: append(kvOf(ConditionsBucket), kvOf(ConditionHistoryBucket)...), Writes: isController},
{State: "calls and their outcomes", Writer: "controller", KeptIn: "key-value " + CallsBucket,
Others: "read by id", Subjects: kvOf(CallsBucket), Writes: isController},
{State: "the hand-act log", Writer: "controller, through the verbs that act", KeptIn: "key-value " + HandActsBucket,
Others: "—", Subjects: kvOf(HandActsBucket), Writes: isController},
{State: "stream definitions and bus permissions", Writer: "controller", KeptIn: "the bus", Others: "—",
// A stream's definition, and a durable consumer's by the API that names it so. Not every
// consumer create: a module watching its own bucket makes and deletes an ordered consumer on the
// bucket's stream (ADR 0201), which defines nothing the mesh keeps.
Subjects: []string{"$JS.API.STREAM.CREATE.>", "$JS.API.STREAM.UPDATE.>", "$JS.API.STREAM.DELETE.>",
"$JS.API.CONSUMER.DURABLE.CREATE.>"},
Writes: isController},
{State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state",
Others: "the controller asks",
Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"},
Writes: holdsSeatOf,
Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"},
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
{State: "a provider's standing", Writer: "the provider", KeptIn: "the provider's events",
Others: "the controller keeps the newest word as a condition",
Subjects: []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"},
Writes: ownModule},
{State: "the operator-channel's open messages", Writer: "the seat's holder", KeptIn: "its own key-value state",
Others: "—"},
{State: "the facts snapshot", Writer: "controller", KeptIn: "the artifact store, facts/latest",
Others: "the build seat reads"},
}
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
// another writer (to-be 45 §1): which state, whose, and the pattern that would make a second writer.
func CheckWriters(p Principal, publish []string) error {
var problems []string
for _, pattern := range publish {
for _, row := range WritersTable {
if row.Writes == nil {
continue
}
for _, subject := range row.Subjects {
if !SubjectsOverlap(pattern, subject) || row.Writes(p, pattern) {
continue
}
problems = append(problems, fmt.Sprintf("%s may publish %s, which writes %s (%s), whose writer is %s",
p.Username(), pattern, row.State, subject, row.Writer))
}
}
}
if len(problems) == 0 {
return nil
}
return fmt.Errorf("a second writer would be granted (novox/hq to-be 45 §1, one writer per piece of state):\n %s",
strings.Join(problems, "\n "))
}
// SubjectsOverlap says some subject matches both patterns: `*` is one token, `>` one or more to the end.
func SubjectsOverlap(a, b string) bool {
x, y := strings.Split(a, "."), strings.Split(b, ".")
for i := 0; ; i++ {
switch {
case i == len(x) && i == len(y):
return true
case i == len(x) || i == len(y):
return false
case x[i] == ">" || y[i] == ">":
return true
case x[i] == "*" || y[i] == "*" || x[i] == y[i]:
continue
default:
return false
}
}
}
+143
View File
@@ -0,0 +1,143 @@
package broker
import (
"slices"
"strings"
"testing"
)
// The writers table, checked (novox/hq to-be 45 §1, ADR 0227 rule 1, "how it is checked"): it is the
// design's table row for row; every row that writes on the bus names its writer; a whole mesh composes
// with one writer per piece of state; and a grant that would make a second writer is refused, naming
// the state and whose it is.
// designRows are the states of to-be 45 §1, in its order. A row added to the design is added here and
// to the table; one dropped from either fails.
var designRows = []string{
"a machine's declaration",
"a machine's applied state and its report",
"the controller lease",
"plans and their tiers",
"conditions",
"calls and their outcomes",
"the hand-act log",
"stream definitions and bus permissions",
"builds and their outcomes",
"a merge announced",
"a provider's standing",
"the operator-channel's open messages",
"the facts snapshot",
}
func TestTheWritersTableIsTheDesigns(t *testing.T) {
var states []string
for _, row := range WritersTable {
states = append(states, row.State)
if row.Writer == "" || row.KeptIn == "" || row.Others == "" {
t.Errorf("%q does not say who writes it, where it is kept and what others do", row.State)
}
if len(row.Subjects) > 0 && row.Writes == nil {
t.Errorf("%q is written on the bus and names no writer among the principals", row.State)
}
}
if !slices.Equal(states, designRows) {
t.Fatalf("the writers table is not to-be 45 §1's:\n have %q\n want %q", states, designRows)
}
}
// A mesh of every kind of principal composes: nobody is granted a second writer's subject.
func TestAWholeMeshComposesWithOneWriterPerState(t *testing.T) {
builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
principals := []Principal{
{Kind: KindController, PasswordHash: "x"},
{Kind: KindNode, Node: "one", PasswordHash: "x"},
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "postgres", Emits: []string{"provisioner.failing", "provisioner.recovered"},
PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
{Module: "gitea", Emits: []string{"pull.merged"}},
{Module: "build-agent", Holds: []Seat{builder}}}},
}
for _, p := range principals {
if _, err := PermissionsFor(p); err != nil {
t.Errorf("%s does not compose: %v", p.Username(), err)
}
}
if _, err := ComposeAccounts(principals); err != nil {
t.Fatalf("the mesh does not compose: %v", err)
}
}
func TestASecondWriterIsRefusedAtComposition(t *testing.T) {
for _, c := range []struct {
name string
p Principal
publish []string
state string
}{
{"the controller publishing what machines say", Principal{Kind: KindController},
[]string{"mesh.control.>"}, "a machine's applied state and its report"},
{"a machine publishing another's report", Principal{Kind: KindNode, Node: "one"},
[]string{"mesh.control.two.report"}, "a machine's applied state and its report"},
{"a machine publishing every machine's", Principal{Kind: KindNode, Node: "one"},
[]string{"mesh.control.*.>"}, "a machine's applied state and its report"},
{"a module writing the lease", Principal{Kind: KindModule, Module: "shop"},
[]string{"$KV.mesh-controller_lease.>"}, "the controller lease"},
{"a module sending a declaration", Principal{Kind: KindModule, Module: "shop"},
[]string{"mesh.node.one.declare"}, "a machine's declaration"},
{"a module defining a stream", Principal{Kind: KindModule, Module: "shop"},
[]string{"$JS.API.>"}, "stream definitions and bus permissions"},
{"a module announcing another forge's merge", Principal{Kind: KindModule, Module: "shop"},
[]string{"mesh.mod.gitea.event.pull.merged"}, "a merge announced"},
{"a module saying a build it did not do", Principal{Kind: KindModule, Module: "shop"},
[]string{"mesh.seat.node-build-agent.event.built"}, "builds and their outcomes"},
} {
t.Run(c.name, func(t *testing.T) {
err := CheckWriters(c.p, c.publish)
if err == nil {
t.Fatalf("%v granted to %s was not refused", c.publish, c.p.Username())
}
if !strings.Contains(err.Error(), c.state) {
t.Fatalf("the refusal does not name %q: %v", c.state, err)
}
})
}
// And the writers themselves are not refused.
for _, ok := range []struct {
p Principal
publish []string
}{
{Principal{Kind: KindNode, Node: "one"}, []string{"mesh.control.one.>"}},
{Principal{Kind: KindController}, []string{"mesh.node.>", "$JS.API.>", "$KV.mesh-controller_lease.>"}},
{Principal{Kind: KindModule, Module: "gitea"}, []string{"mesh.mod.gitea.event.pull.merged"}},
{Principal{Kind: KindModule, Module: "shop"}, []string{"$JS.API.CONSUMER.CREATE.KV_shop_carts.>"}},
} {
if err := CheckWriters(ok.p, ok.publish); err != nil {
t.Errorf("a writer was refused its own: %v", err)
}
}
}
func TestSubjectsOverlap(t *testing.T) {
for _, c := range []struct {
a, b string
want bool
}{
{"mesh.control.>", "mesh.control.*.report", true},
{"mesh.control.one.>", "mesh.control.*.report", true},
{"mesh.control.one.alive", "mesh.control.*.report", false},
{"mesh.control.*", "mesh.control.*.report", false},
{"$JS.API.>", "$JS.API.STREAM.CREATE.>", true},
{"$JS.API.CONSUMER.CREATE.KV_x.>", "$JS.API.STREAM.CREATE.>", false},
{"a.b", "a.b", true},
{"a.b", "a.b.c", false},
{"a.>", "a", false},
} {
if got := SubjectsOverlap(c.a, c.b); got != c.want || SubjectsOverlap(c.b, c.a) != c.want {
t.Errorf("%s ~ %s: %v, want %v", c.a, c.b, got, c.want)
}
}
}