diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 7d855bc9..ecec2bba 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -823,6 +823,11 @@ func PermissionsFor(p Principal) (Permissions, error) { } case KindModule: + // 0. The watchman alone hears who connects and disconnects, and asks who is connected (ADR 0431 §2). + if p.Module == WatchmanModule { + sub = append(sub, BusConnectSubject, BusDisconnectSubject) + pub = append(pub, BusConnzSubject) + } // 1. Its own namespace: it publishes its events there and serves its tools there. Nothing // else may publish into it, so an event's source is a fact the server enforces rather // than a claim in the body (design 29 §2). @@ -1447,7 +1452,9 @@ func ComposeAccounts(principals []Principal) (string, error) { // JetStream is enabled per account once accounts exist at all: with only the global block set, // a user in MESH is told "JetStream not enabled for account" the first time it binds a // consumer, which is the first thing every host does (2026-09-28). - b.WriteString("accounts {\n MESH {\n jetstream: enabled\n users = [\n") + b.WriteString("accounts {\n MESH {\n jetstream: enabled\n") + b.WriteString(meshSystemImports) + b.WriteString(" users = [\n") for _, p := range sorted { perms, err := PermissionsFor(p) if err != nil { @@ -1468,10 +1475,45 @@ func ComposeAccounts(principals []Principal) (string, error) { } b.WriteString(" } }\n") } - b.WriteString(" ]\n }\n}\n") + b.WriteString(" ]\n }\n") + b.WriteString(systemAccount) + b.WriteString("}\n") return b.String(), nil } +// WatchmanModule is the one module whose principal hears the bus's connect and disconnect events and asks its list of +// connections (novox/hq ADR 0431 §2, issue 1147): the watchman of the repository mesh-delivery, which writes presence +// from them. No other principal is granted the subjects below, and nothing is a wildcard over them. +const WatchmanModule = "mesh-delivery-watchman" + +// The bus's own events, imported into MESH from the default system account `$SYS` under subjects of MESH's own +// (novox/hq ADR 0431 §1, issue 1147): the server says an account's connections opening and closing only to the system +// account, and the list of an account's connections only to a request made there. `$SYS` holds no user, so nobody +// gains its other requests and events; it exports exactly these three, to MESH alone. +const ( + BusConnectSubject = "mesh.bus.connect" + BusDisconnectSubject = "mesh.bus.disconnect" + BusConnzSubject = "mesh.bus.connz" +) + +// meshSystemImports is what MESH imports from $SYS: the two event streams and the one service. +const meshSystemImports = " imports = [\n" + + " { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.CONNECT\" }, to: \"" + BusConnectSubject + "\" }\n" + + " { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.DISCONNECT\" }, to: \"" + BusDisconnectSubject + "\" }\n" + + " { service: { account: \"$SYS\", subject: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\" }, to: \"" + BusConnzSubject + "\" }\n" + + " ]\n" + +// systemAccount is the account `$SYS`, the server's default system account by its own name (a config that declares an +// account of that name makes it the system account), with no user and the three exports to MESH. Quoted, because an +// unquoted `$` begins a variable in the server's config grammar. +const systemAccount = " \"$SYS\" {\n" + + " exports = [\n" + + " { stream: \"$SYS.ACCOUNT.MESH.CONNECT\", accounts: [\"MESH\"] }\n" + + " { stream: \"$SYS.ACCOUNT.MESH.DISCONNECT\", accounts: [\"MESH\"] }\n" + + " { service: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\", accounts: [\"MESH\"] }\n" + + " ]\n" + + " }\n" + // unique is a sorted list with each subject once. Two carried modules holding seats with the same // verb, or the runtime module itself carried beside the others, would otherwise write a grant twice // — harmless to the server, and noise in a file that is read as the mesh's authority model. diff --git a/internal/broker/nats_golden_test.go b/internal/broker/nats_golden_test.go index 01571741..55c6fbd7 100644 --- a/internal/broker/nats_golden_test.go +++ b/internal/broker/nats_golden_test.go @@ -28,6 +28,8 @@ func TestTheComposedConfigMatchesTheGolden(t *testing.T) { Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"}, {Kind: KindModule, Node: "two", Module: "audit", Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"}, + // The watchman alone hears the bus's connections (novox/hq ADR 0431 §2, issue 1147). + {Kind: KindModule, Node: "one", Module: WatchmanModule, PasswordHash: "$2a$11$wwwwwwwwwwwwwwwwwwwwww"}, // The bus's own module: the snapshot API and its inbox, nothing else (novox/hq ADR 0235). {Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true, Serves: []string{"nats_streams"}, PasswordHash: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb"}, diff --git a/internal/broker/system_exports_live_test.go b/internal/broker/system_exports_live_test.go new file mode 100644 index 00000000..48613678 --- /dev/null +++ b/internal/broker/system_exports_live_test.go @@ -0,0 +1,164 @@ +//go:build integration + +package broker + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "golang.org/x/crypto/bcrypt" +) + +// **The system account's exports reach MESH on a reload, with no restart** (novox/hq ADR 0431 §3, issue 1147, +// 1147/AC3): the embedded pinned server starts with today's composed accounts (no `$SYS`, no import); the new file is +// written and the server reloaded; the reload succeeds; a user with the watchman's grants hears another user's connect +// and disconnect and gets an answer to `mesh.bus.connz` that lists the open connections; another module's user is +// refused a subscribe on `mesh.bus.connect`. +func TestNatsTheSystemAccountsExportsReachMeshOnAReload(t *testing.T) { // 1147/AC3 + hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + users, err := Users(Records{Nodes: []string{"one"}, Assigned: map[string][]Declared{ + "one": {{Module: RuntimeModule}, {Module: WatchmanModule}, {Module: "gitea"}, {Module: "shop"}}, + }}) + if err != nil { + t.Fatal(err) + } + for i := range users { + users[i].PasswordHash = string(hash) + } + accounts, err := ComposeAccounts(users) + if err != nil { + t.Fatal(err) + } + // Today's file: the same accounts without the system account's exports and MESH's imports. + today := strings.Replace(strings.Replace(accounts, meshSystemImports, "", 1), systemAccount, "", 1) + if today == accounts || strings.Contains(today, "$SYS") { + t.Fatal("today's composed file could not be derived from the new one") + } + dir := t.TempDir() + conf := filepath.Join(dir, "bus.conf") + write := func(accounts string) { + t.Helper() + if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+dir+`/js"`+" }\n"+accounts), 0o600); err != nil { + t.Fatal(err) + } + } + write(today) + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatalf("today's accounts do not parse: %v", err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + + connect := func(user string) (*nats.Conn, chan error) { + t.Helper() + refused := make(chan error, 8) + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err })) + if err != nil { + t.Fatalf("%s cannot connect: %v", user, err) + } + t.Cleanup(nc.Close) + return nc, refused + } + // A connection open across the reload: the reload must not drop it. + shop, _ := connect("one.shop") + + write(accounts) + if err := s.Reload(); err != nil { + t.Fatalf("the server does not take the system account's exports on a reload: %v", err) + } + if !shop.IsConnected() { + t.Error("the reload dropped a connection that was open") + } + + watchman, _ := connect("one." + WatchmanModule) + connects := make(chan *nats.Msg, 16) + disconnects := make(chan *nats.Msg, 16) + if _, err := watchman.ChanSubscribe(BusConnectSubject, connects); err != nil { + t.Fatal(err) + } + if _, err := watchman.ChanSubscribe(BusDisconnectSubject, disconnects); err != nil { + t.Fatal(err) + } + if err := watchman.Flush(); err != nil { + t.Fatal(err) + } + // Another user connects and disconnects: the watchman hears both, and the events name that user. + gitea, _ := connect("one.gitea") + hears := func(ch chan *nats.Msg, what string) { + t.Helper() + select { + case m := <-ch: + if !strings.Contains(string(m.Data), "one.gitea") { + t.Errorf("the %s event does not name the user that connected: %s", what, m.Data) + } + case <-time.After(5 * time.Second): + t.Errorf("the watchman did not hear another user's %s", what) + } + } + hears(connects, "connect") + // The answer to the connections request lists the open connections, the shop's among them. + reply, err := watchman.Request(BusConnzSubject, []byte(`{}`), 5*time.Second) + if err != nil { + t.Fatalf("mesh.bus.connz is not answered: %v", err) + } + var answer struct { + Data struct { + Connections []struct { + AuthorizedUser string `json:"authorized_user"` + } `json:"connections"` + } `json:"data"` + } + if err := json.Unmarshal(reply.Data, &answer); err != nil { + t.Fatalf("the connections answer is not JSON: %v: %s", err, reply.Data) + } + var listed bool + for _, c := range answer.Data.Connections { + listed = listed || c.AuthorizedUser == "one.shop" + } + if !listed { + t.Errorf("the connections answer does not list the shop's connection: %s", reply.Data) + } + gitea.Close() + hears(disconnects, "disconnect") + + // Another module's user is refused a subscribe on the events and a publish on the request. + other, refused := connect("one.shop") + for _, act := range []func() error{ + func() error { _, err := other.SubscribeSync(BusConnectSubject); return err }, + func() error { _, err := other.SubscribeSync(BusDisconnectSubject); return err }, + func() error { return other.Publish(BusConnzSubject, []byte(`{}`)) }, + } { + if err := act(); err != nil { + t.Fatal(err) + } + _ = other.Flush() + select { + case err := <-refused: + if !errors.Is(err, nats.ErrPermissionViolation) { + t.Errorf("the server said %v, want a permissions violation", err) + } + case <-time.After(3 * time.Second): + t.Error("the server did not refuse another module's user the bus's connection subjects") + } + } +} diff --git a/internal/broker/system_exports_test.go b/internal/broker/system_exports_test.go new file mode 100644 index 00000000..ebce219e --- /dev/null +++ b/internal/broker/system_exports_test.go @@ -0,0 +1,145 @@ +package broker + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nats-io/nats-server/v2/server" +) + +// 1147/AC1: the composed accounts file declares $SYS with no user, exporting only the connect and disconnect events of +// MESH and the request listing MESH's connections, and MESH imports exactly those three as mesh.bus.connect, +// mesh.bus.disconnect and mesh.bus.connz (novox/hq ADR 0431 §1); the server's own grammar takes it with $SYS as its +// system account; the golden file shows it (TestTheComposedConfigMatchesTheGolden). +func TestTheComposedAccountsExportMeshsConnectionEventsFromTheSystemAccount(t *testing.T) { // 1147/AC1 + text, err := ComposeAccounts([]Principal{{Kind: KindNode, Node: "one", PasswordHash: "x"}}) + if err != nil { + t.Fatal(err) + } + sys := between(text, `"$SYS" {`, "\n }\n") + if sys == "" { + t.Fatalf("no account $SYS in the composed accounts:\n%s", text) + } + if strings.Contains(sys, "users") || strings.Contains(sys, "user:") { + t.Errorf("$SYS holds a user:\n%s", sys) + } + for _, want := range []string{ + `{ stream: "$SYS.ACCOUNT.MESH.CONNECT", accounts: ["MESH"] }`, + `{ stream: "$SYS.ACCOUNT.MESH.DISCONNECT", accounts: ["MESH"] }`, + `{ service: "$SYS.REQ.ACCOUNT.MESH.CONNZ", accounts: ["MESH"] }`, + } { + if !strings.Contains(sys, want) { + t.Errorf("$SYS does not export %s:\n%s", want, sys) + } + } + if n := strings.Count(sys, "{ stream:") + strings.Count(sys, "{ service:"); n != 3 { + t.Errorf("$SYS exports %d things, want exactly 3:\n%s", n, sys) + } + mesh := between(text, " MESH {", " users = [") + for _, want := range []string{ + `{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.CONNECT" }, to: "mesh.bus.connect" }`, + `{ stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.DISCONNECT" }, to: "mesh.bus.disconnect" }`, + `{ service: { account: "$SYS", subject: "$SYS.REQ.ACCOUNT.MESH.CONNZ" }, to: "mesh.bus.connz" }`, + } { + if !strings.Contains(mesh, want) { + t.Errorf("MESH does not import %s:\n%s", want, mesh) + } + } + if n := strings.Count(mesh, "{ stream:") + strings.Count(mesh, "{ service:"); n != 3 { + t.Errorf("MESH imports %d things, want exactly 3:\n%s", n, mesh) + } + + // The pinned server's grammar takes it, and makes $SYS its system account. + path := filepath.Join(t.TempDir(), "accounts.conf") + if err := os.WriteFile(path, []byte(text), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(path) + if err != nil { + t.Fatalf("the server refuses the composed accounts: %v", err) + } + var found bool + for _, a := range opts.Accounts { + if a.Name != "$SYS" { + continue + } + found = true + if !a.IsExportService("$SYS.REQ.ACCOUNT.MESH.CONNZ") { + t.Error("$SYS does not export the connections request as a service") + } + } + if !found { + t.Error("the server read no account $SYS") + } + for _, u := range opts.Users { + if u.Account != nil && u.Account.Name == "$SYS" { + t.Errorf("user %s is in $SYS", u.Username) + } + } +} + +// between is the text from the first start to the first end after it, or empty. +func between(s, start, end string) string { + i := strings.Index(s, start) + if i < 0 { + return "" + } + rest := s[i:] + j := strings.Index(rest[len(start):], end) + if j < 0 { + return rest + } + return rest[:len(start)+j] +} + +// 1147/AC2: only mesh-delivery-watchman's principal may subscribe to mesh.bus.connect and mesh.bus.disconnect and +// publish to mesh.bus.connz; no other principal's grant covers them by a wildcard (ADR 0431 §2). +func TestOnlyTheWatchmanHearsTheBusConnectionEvents(t *testing.T) { // 1147/AC2 + builder := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"}, Emits: []string{"built"}} + watchman := Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x"} + others := []Principal{ + {Kind: KindController, PasswordHash: "x"}, + {Kind: KindNode, Node: "one", PasswordHash: "x"}, + {Kind: KindEnrolment, Node: "two", PasswordHash: "x"}, + {Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"}, + {Kind: KindProvisioner, Node: "one", PasswordHash: "x"}, + {Kind: KindModule, Node: "one", Module: "gitea", Emits: []string{"pull.merged"}, PasswordHash: "x"}, + {Kind: KindModule, Node: "one", Module: "mesh-delivery-view", PasswordHash: "x", + Reads: []string{"node-engine.have"}, Consumes: []string{"mesh-delivery-watchman.presence"}}, + {Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{builder}, PasswordHash: "x"}, + {Kind: KindModule, Node: "one", Module: "nats", SnapshotsTheBus: true, PasswordHash: "x"}, + {Kind: KindNodeTools, Node: "one", Module: RuntimeModule, PasswordHash: "x", Carries: []Declared{ + {Module: "mesh-delivery-watchman"}, {Module: "build-agent", Holds: []Seat{builder}}}}, + } + heard := []string{"mesh.bus.connect", "mesh.bus.disconnect"} + perms, err := PermissionsFor(watchman) + if err != nil { + t.Fatal(err) + } + for _, s := range heard { + has(t, perms.Subscribe, s) + } + has(t, perms.Publish, "mesh.bus.connz") + for _, s := range append(heard, "mesh.bus.connz") { + for _, p := range perms.Subscribe { + if p != s && SubjectsOverlap(p, s) && !strings.HasPrefix(p, "_INBOX.") { + t.Errorf("the watchman's subscription %q reaches %s beyond naming it", p, s) + } + } + } + for _, p := range others { + perms, err := PermissionsFor(p) + if err != nil { + t.Fatalf("%s: %v", p.Username(), err) + } + for _, s := range append(heard, "mesh.bus.connz") { + for _, g := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) { + if SubjectsOverlap(g, s) { + t.Errorf("%s is granted %q, which covers %s", p.Username(), g, s) + } + } + } + } +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 0c3865fb..f0d25f8d 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -22,6 +22,11 @@ jetstream { accounts { MESH { jetstream: enabled + imports = [ + { stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.CONNECT" }, to: "mesh.bus.connect" } + { stream: { account: "$SYS", subject: "$SYS.ACCOUNT.MESH.DISCONNECT" }, to: "mesh.bus.disconnect" } + { service: { account: "$SYS", subject: "$SYS.REQ.ACCOUNT.MESH.CONNZ" }, to: "mesh.bus.connz" } + ] users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } @@ -37,6 +42,11 @@ accounts { subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] } allow_responses: { max: 1, ttl: "1m" } } } + { user: "one.mesh-delivery-watchman", password: "$2a$11$wwwwwwwwwwwwwwwwwwwwww", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.one_mesh-delivery-watchman.>", "$JS.API.CONSUMER.INFO.EVENTS.one_mesh-delivery-watchman", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_mesh-delivery-watchman", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.mesh-delivery-watchman", "mesh.bus.connz"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.mesh-delivery-watchman", "$SRV.INFO.mesh-delivery-watchman.>", "$SRV.PING", "$SRV.PING.mesh-delivery-watchman", "$SRV.PING.mesh-delivery-watchman.>", "$SRV.STATS", "$SRV.STATS.mesh-delivery-watchman", "$SRV.STATS.mesh-delivery-watchman.>", "_INBOX.one.mesh-delivery-watchman.>", "mesh.assignment.one.mesh-delivery-watchman", "mesh.bus.connect", "mesh.bus.disconnect", "mesh.mod.mesh-delivery-watchman.call.>", "mesh.mod.mesh-delivery-watchman.tool.>"] } + allow_responses: { max: 1, ttl: "1m" } + } } { user: "one.nats", password: "$2a$11$bbbbbbbbbbbbbbbbbbbbbb", permissions: { publish: { allow: ["$JS.API.STREAM.INFO.*", "$JS.API.STREAM.NAMES", "$JS.API.STREAM.SNAPSHOT.*", "$JS.SNAPSHOT.ACK.>"] } subscribe: { allow: ["_INBOX.one.nats.>"] } @@ -58,4 +68,11 @@ accounts { } } ] } + "$SYS" { + exports = [ + { stream: "$SYS.ACCOUNT.MESH.CONNECT", accounts: ["MESH"] } + { stream: "$SYS.ACCOUNT.MESH.DISCONNECT", accounts: ["MESH"] } + { service: "$SYS.REQ.ACCOUNT.MESH.CONNZ", accounts: ["MESH"] } + ] + } }