The bus's system account exports MESH's connect and disconnect events and connections request, heard only by the watchman (issue 1147)

This commit is contained in:
2026-10-18 19:07:06 +02:00
parent 661e8eff2f
commit 8c8bc5021e
5 changed files with 372 additions and 2 deletions
+44 -2
View File
@@ -823,6 +823,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
}
case KindModule:
// 0. The watchman alone hears who connects and disconnects, and asks who is connected (ADR 0431 §2).
if p.Module == WatchmanModule {
sub = append(sub, BusConnectSubject, BusDisconnectSubject)
pub = append(pub, BusConnzSubject)
}
// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
// else may publish into it, so an event's source is a fact the server enforces rather
// than a claim in the body (design 29 §2).
@@ -1447,7 +1452,9 @@ func ComposeAccounts(principals []Principal) (string, error) {
// JetStream is enabled per account once accounts exist at all: with only the global block set,
// a user in MESH is told "JetStream not enabled for account" the first time it binds a
// consumer, which is the first thing every host does (2026-09-28).
b.WriteString("accounts {\n MESH {\n jetstream: enabled\n users = [\n")
b.WriteString("accounts {\n MESH {\n jetstream: enabled\n")
b.WriteString(meshSystemImports)
b.WriteString(" users = [\n")
for _, p := range sorted {
perms, err := PermissionsFor(p)
if err != nil {
@@ -1468,10 +1475,45 @@ func ComposeAccounts(principals []Principal) (string, error) {
}
b.WriteString(" } }\n")
}
b.WriteString(" ]\n }\n}\n")
b.WriteString(" ]\n }\n")
b.WriteString(systemAccount)
b.WriteString("}\n")
return b.String(), nil
}
// WatchmanModule is the one module whose principal hears the bus's connect and disconnect events and asks its list of
// connections (novox/hq ADR 0431 §2, issue 1147): the watchman of the repository mesh-delivery, which writes presence
// from them. No other principal is granted the subjects below, and nothing is a wildcard over them.
const WatchmanModule = "mesh-delivery-watchman"
// The bus's own events, imported into MESH from the default system account `$SYS` under subjects of MESH's own
// (novox/hq ADR 0431 §1, issue 1147): the server says an account's connections opening and closing only to the system
// account, and the list of an account's connections only to a request made there. `$SYS` holds no user, so nobody
// gains its other requests and events; it exports exactly these three, to MESH alone.
const (
BusConnectSubject = "mesh.bus.connect"
BusDisconnectSubject = "mesh.bus.disconnect"
BusConnzSubject = "mesh.bus.connz"
)
// meshSystemImports is what MESH imports from $SYS: the two event streams and the one service.
const meshSystemImports = " imports = [\n" +
" { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.CONNECT\" }, to: \"" + BusConnectSubject + "\" }\n" +
" { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.DISCONNECT\" }, to: \"" + BusDisconnectSubject + "\" }\n" +
" { service: { account: \"$SYS\", subject: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\" }, to: \"" + BusConnzSubject + "\" }\n" +
" ]\n"
// systemAccount is the account `$SYS`, the server's default system account by its own name (a config that declares an
// account of that name makes it the system account), with no user and the three exports to MESH. Quoted, because an
// unquoted `$` begins a variable in the server's config grammar.
const systemAccount = " \"$SYS\" {\n" +
" exports = [\n" +
" { stream: \"$SYS.ACCOUNT.MESH.CONNECT\", accounts: [\"MESH\"] }\n" +
" { stream: \"$SYS.ACCOUNT.MESH.DISCONNECT\", accounts: [\"MESH\"] }\n" +
" { service: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\", accounts: [\"MESH\"] }\n" +
" ]\n" +
" }\n"
// unique is a sorted list with each subject once. Two carried modules holding seats with the same
// verb, or the runtime module itself carried beside the others, would otherwise write a grant twice
// — harmless to the server, and noise in a file that is read as the mesh's authority model.
+2
View File
@@ -28,6 +28,8 @@ func TestTheComposedConfigMatchesTheGolden(t *testing.T) {
Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"},
{Kind: KindModule, Node: "two", Module: "audit",
Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"},
// The watchman alone hears the bus's connections (novox/hq ADR 0431 §2, issue 1147).
{Kind: KindModule, Node: "one", Module: WatchmanModule, PasswordHash: "$2a$11$wwwwwwwwwwwwwwwwwwwwww"},
// The bus's own module: the snapshot API and its inbox, nothing else (novox/hq ADR 0235).
{Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true,
Serves: []string{"nats_streams"}, PasswordHash: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb"},
+164
View File
@@ -0,0 +1,164 @@
//go:build integration
package broker
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"golang.org/x/crypto/bcrypt"
)
// **The system account's exports reach MESH on a reload, with no restart** (novox/hq ADR 0431 §3, issue 1147,
// 1147/AC3): the embedded pinned server starts with today's composed accounts (no `$SYS`, no import); the new file is
// written and the server reloaded; the reload succeeds; a user with the watchman's grants hears another user's connect
// and disconnect and gets an answer to `mesh.bus.connz` that lists the open connections; another module's user is
// refused a subscribe on `mesh.bus.connect`.
func TestNatsTheSystemAccountsExportsReachMeshOnAReload(t *testing.T) { // 1147/AC3
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
users, err := Users(Records{Nodes: []string{"one"}, Assigned: map[string][]Declared{
"one": {{Module: RuntimeModule}, {Module: WatchmanModule}, {Module: "gitea"}, {Module: "shop"}},
}})
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = string(hash)
}
accounts, err := ComposeAccounts(users)
if err != nil {
t.Fatal(err)
}
// Today's file: the same accounts without the system account's exports and MESH's imports.
today := strings.Replace(strings.Replace(accounts, meshSystemImports, "", 1), systemAccount, "", 1)
if today == accounts || strings.Contains(today, "$SYS") {
t.Fatal("today's composed file could not be derived from the new one")
}
dir := t.TempDir()
conf := filepath.Join(dir, "bus.conf")
write := func(accounts string) {
t.Helper()
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+dir+`/js"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
}
write(today)
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("today's accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(s.Shutdown)
connect := func(user string) (*nats.Conn, chan error) {
t.Helper()
refused := make(chan error, 8)
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
if err != nil {
t.Fatalf("%s cannot connect: %v", user, err)
}
t.Cleanup(nc.Close)
return nc, refused
}
// A connection open across the reload: the reload must not drop it.
shop, _ := connect("one.shop")
write(accounts)
if err := s.Reload(); err != nil {
t.Fatalf("the server does not take the system account's exports on a reload: %v", err)
}
if !shop.IsConnected() {
t.Error("the reload dropped a connection that was open")
}
watchman, _ := connect("one." + WatchmanModule)
connects := make(chan *nats.Msg, 16)
disconnects := make(chan *nats.Msg, 16)
if _, err := watchman.ChanSubscribe(BusConnectSubject, connects); err != nil {
t.Fatal(err)
}
if _, err := watchman.ChanSubscribe(BusDisconnectSubject, disconnects); err != nil {
t.Fatal(err)
}
if err := watchman.Flush(); err != nil {
t.Fatal(err)
}
// Another user connects and disconnects: the watchman hears both, and the events name that user.
gitea, _ := connect("one.gitea")
hears := func(ch chan *nats.Msg, what string) {
t.Helper()
select {
case m := <-ch:
if !strings.Contains(string(m.Data), "one.gitea") {
t.Errorf("the %s event does not name the user that connected: %s", what, m.Data)
}
case <-time.After(5 * time.Second):
t.Errorf("the watchman did not hear another user's %s", what)
}
}
hears(connects, "connect")
// The answer to the connections request lists the open connections, the shop's among them.
reply, err := watchman.Request(BusConnzSubject, []byte(`{}`), 5*time.Second)
if err != nil {
t.Fatalf("mesh.bus.connz is not answered: %v", err)
}
var answer struct {
Data struct {
Connections []struct {
AuthorizedUser string `json:"authorized_user"`
} `json:"connections"`
} `json:"data"`
}
if err := json.Unmarshal(reply.Data, &answer); err != nil {
t.Fatalf("the connections answer is not JSON: %v: %s", err, reply.Data)
}
var listed bool
for _, c := range answer.Data.Connections {
listed = listed || c.AuthorizedUser == "one.shop"
}
if !listed {
t.Errorf("the connections answer does not list the shop's connection: %s", reply.Data)
}
gitea.Close()
hears(disconnects, "disconnect")
// Another module's user is refused a subscribe on the events and a publish on the request.
other, refused := connect("one.shop")
for _, act := range []func() error{
func() error { _, err := other.SubscribeSync(BusConnectSubject); return err },
func() error { _, err := other.SubscribeSync(BusDisconnectSubject); return err },
func() error { return other.Publish(BusConnzSubject, []byte(`{}`)) },
} {
if err := act(); err != nil {
t.Fatal(err)
}
_ = other.Flush()
select {
case err := <-refused:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("the server said %v, want a permissions violation", err)
}
case <-time.After(3 * time.Second):
t.Error("the server did not refuse another module's user the bus's connection subjects")
}
}
}
+145
View File
@@ -0,0 +1,145 @@
package broker
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/nats-io/nats-server/v2/server"
)
// 1147/AC1: the composed accounts file declares $SYS with no user, exporting only the connect and disconnect events of
// MESH and the request listing MESH's connections, and MESH imports exactly those three as mesh.bus.connect,
// mesh.bus.disconnect and mesh.bus.connz (novox/hq ADR 0431 §1); the server's own grammar takes it with $SYS as its
// system account; the golden file shows it (TestTheComposedConfigMatchesTheGolden).
func TestTheComposedAccountsExportMeshsConnectionEventsFromTheSystemAccount(t *testing.T) { // 1147/AC1
text, err := ComposeAccounts([]Principal{{Kind: KindNode, Node: "one", PasswordHash: "x"}})
if err != nil {
t.Fatal(err)
}
sys := between(text, `"$SYS" {`, "\n }\n")
if sys == "" {
t.Fatalf("no account $SYS in the composed accounts:\n%s", text)
}
if strings.Contains(sys, "users") || strings.Contains(sys, "user:") {
t.Errorf("$SYS holds a user:\n%s", sys)
}
for _, want := range []string{
`{ stream: "$SYS.ACCOUNT.MESH.CONNECT", accounts: ["MESH"] }`,
`{ stream: "$SYS.ACCOUNT.MESH.DISCONNECT", accounts: ["MESH"] }`,
`{ service: "$SYS.REQ.ACCOUNT.MESH.CONNZ", accounts: ["MESH"] }`,
} {
if !strings.Contains(sys, want) {
t.Errorf("$SYS does not export %s:\n%s", want, sys)
}
}
if n := strings.Count(sys, "{ stream:") + strings.Count(sys, "{ service:"); n != 3 {
t.Errorf("$SYS exports %d things, want exactly 3:\n%s", n, sys)
}
mesh := between(text, " MESH {", " users = [")
for _, want := range []string{
`{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.CONNECT" }, to: "mesh.bus.connect" }`,
`{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.DISCONNECT" }, to: "mesh.bus.disconnect" }`,
`{ service: { account: "$SYS", subject: "$SYS.REQ.ACCOUNT.MESH.CONNZ" }, to: "mesh.bus.connz" }`,
} {
if !strings.Contains(mesh, want) {
t.Errorf("MESH does not import %s:\n%s", want, mesh)
}
}
if n := strings.Count(mesh, "{ stream:") + strings.Count(mesh, "{ service:"); n != 3 {
t.Errorf("MESH imports %d things, want exactly 3:\n%s", n, mesh)
}
// The pinned server's grammar takes it, and makes $SYS its system account.
path := filepath.Join(t.TempDir(), "accounts.conf")
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(path)
if err != nil {
t.Fatalf("the server refuses the composed accounts: %v", err)
}
var found bool
for _, a := range opts.Accounts {
if a.Name != "$SYS" {
continue
}
found = true
if !a.IsExportService("$SYS.REQ.ACCOUNT.MESH.CONNZ") {
t.Error("$SYS does not export the connections request as a service")
}
}
if !found {
t.Error("the server read no account $SYS")
}
for _, u := range opts.Users {
if u.Account != nil && u.Account.Name == "$SYS" {
t.Errorf("user %s is in $SYS", u.Username)
}
}
}
// between is the text from the first start to the first end after it, or empty.
func between(s, start, end string) string {
i := strings.Index(s, start)
if i < 0 {
return ""
}
rest := s[i:]
j := strings.Index(rest[len(start):], end)
if j < 0 {
return rest
}
return rest[:len(start)+j]
}
// 1147/AC2: only mesh-delivery-watchman's principal may subscribe to mesh.bus.connect and mesh.bus.disconnect and
// publish to mesh.bus.connz; no other principal's grant covers them by a wildcard (ADR 0431 §2).
func TestOnlyTheWatchmanHearsTheBusConnectionEvents(t *testing.T) { // 1147/AC2
builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built"}}
watchman := Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x"}
others := []Principal{
{Kind: KindController, PasswordHash: "x"},
{Kind: KindNode, Node: "one", PasswordHash: "x"},
{Kind: KindEnrolment, Node: "two", PasswordHash: "x"},
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
{Kind: KindProvisioner, Node: "one", PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "mesh-delivery-view", PasswordHash: "x",
Reads: []string{"node-engine.have"}, Consumes: []string{"mesh-delivery-watchman.presence"}},
{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"},
{Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true, PasswordHash: "x"},
{Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{
{Module: "mesh-delivery-watchman"}, {Module: "build-agent", Holds: []Seat{builder}}}},
}
heard := []string{"mesh.bus.connect", "mesh.bus.disconnect"}
perms, err := PermissionsFor(watchman)
if err != nil {
t.Fatal(err)
}
for _, s := range heard {
has(t, perms.Subscribe, s)
}
has(t, perms.Publish, "mesh.bus.connz")
for _, s := range append(heard, "mesh.bus.connz") {
for _, p := range perms.Subscribe {
if p != s && SubjectsOverlap(p, s) && !strings.HasPrefix(p, "_INBOX.") {
t.Errorf("the watchman's subscription %q reaches %s beyond naming it", p, s)
}
}
}
for _, p := range others {
perms, err := PermissionsFor(p)
if err != nil {
t.Fatalf("%s: %v", p.Username(), err)
}
for _, s := range append(heard, "mesh.bus.connz") {
for _, g := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
if SubjectsOverlap(g, s) {
t.Errorf("%s is granted %q, which covers %s", p.Username(), g, s)
}
}
}
}
}
+17
View File
@@ -22,6 +22,11 @@ jetstream {
accounts {
MESH {
jetstream: enabled
imports = [
{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.CONNECT" }, to: "mesh.bus.connect" }
{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.DISCONNECT" }, to: "mesh.bus.disconnect" }
{ service: { account: "$SYS", subject: "$SYS.REQ.ACCOUNT.MESH.CONNZ" }, to: "mesh.bus.connz" }
]
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
@@ -37,6 +42,11 @@ accounts {
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: {
publish: { allow: ["$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.mesh-delivery-watchman", "mesh.bus.connz"] }
subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.mesh-delivery-watchman", "$SRV.INFO.mesh-delivery-watchman.>", "$SRV.PING", "$SRV.PING.mesh-delivery-watchman", "$SRV.PING.mesh-delivery-watchman.>", "$SRV.STATS", "$SRV.STATS.mesh-delivery-watchman", "$SRV.STATS.mesh-delivery-watchman.>", "_INBOX.one.mesh-delivery-watchman.>", "mesh.assignment.one.mesh-delivery-watchman", "mesh.bus.connect", "mesh.bus.disconnect", "mesh.mod.mesh-delivery-watchman.call.>", "mesh.mod.mesh-delivery-watchman.tool.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: {
publish: { allow: ["$JS.API.STREAM.INFO.*", "$JS.API.STREAM.NAMES", "$JS.API.STREAM.SNAPSHOT.*", "$JS.SNAPSHOT.ACK.>"] }
subscribe: { allow: ["_INBOX.one.nats.>"] }
@@ -58,4 +68,11 @@ accounts {
} }
]
}
"$SYS" {
exports = [
{ stream: "$SYS.ACCOUNT.MESH.CONNECT", accounts: ["MESH"] }
{ stream: "$SYS.ACCOUNT.MESH.DISCONNECT", accounts: ["MESH"] }
{ service: "$SYS.REQ.ACCOUNT.MESH.CONNZ", accounts: ["MESH"] }
]
}
}