From fb87f9f7d6dcaa3dd8e708573c039d5f38ea3005 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 19:34:14 +0200 Subject: [PATCH 01/39] The mesh-broker seat delivers nothing The bus is the only broker (novox/hq ADR 0117): messaging is subjects on it, scoped by a module's own emits/consumes, not a server handed out as a provision. So the seat joins mesh-controller and the-catalogue in delivering no interface. Full suite green. --- internal/catalogue/seats.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 2475e06..591c8be 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -35,7 +35,11 @@ type Seat struct { var seats = []Seat{ {Name: "mesh-controller", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"}, {Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079"}, - {Name: "mesh-broker", Scope: ScopeMesh, Delivers: "amqp", Decision: "novox/hq ADR 0079"}, + // The bus is the only broker (novox/hq ADR 0117): a module's messaging is subjects on it, + // scoped by its own emits/consumes, not a server handed out as a provision. So this seat + // delivers nothing, like mesh-controller and the-catalogue. The `amqp` interface — a private + // broker per consumer — retires with the compatibility broker. + {Name: "mesh-broker", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"}, {Name: "the-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"}, {Name: "the-catalogue", Scope: ScopeMesh, Decision: "novox/hq ADR 0110"}, {Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"}, -- 2.54.0 From c753f9d5c00753484cfd07ffef48213d43243167 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 20:58:49 +0200 Subject: [PATCH 02/39] Compose the bus's accounts instead of calling a management API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 1.3 of novox/hq ADR 0116. On AMQP an account was an HTTP call; on NATS it is text the controller composes and the server reloads (ADR 0106). Pure, so the mesh's whole authority model is testable as strings. NATS closes a gap management.go recorded rather than hid: LavinMQ has no topic permissions, so an emitter was granted the events exchange whole and ADR 0042's origin reservation was "stamped by the sdk, not enforced here". Per-subject permissions make it the server's refusal. Two things found by composing a real file rather than reading the design: - a scoped inbox leaves a responder unable to reply, because the answer goes to the caller's inbox. allow_responses is the answer — one reply to the subject of a message actually received — and only principals that serve are granted it. Recorded in design 25 §4. - composition must be deterministic: the module's entrypoint reloads on the file's digest, so an order-dependent composer would reload the whole bus on every controller restart. Covered by a test. The golden fixture is the exact text `nats-server -t` accepts, so the syntax is the server's rather than one we invented. --- internal/broker/nats.go | 306 +++++++++++++++++++++++++ internal/broker/nats_golden_test.go | 49 ++++ internal/broker/nats_test.go | 166 ++++++++++++++ internal/broker/testdata/composed.conf | 50 ++++ 4 files changed, 571 insertions(+) create mode 100644 internal/broker/nats.go create mode 100644 internal/broker/nats_golden_test.go create mode 100644 internal/broker/nats_test.go create mode 100644 internal/broker/testdata/composed.conf diff --git a/internal/broker/nats.go b/internal/broker/nats.go new file mode 100644 index 0000000..62d2bf0 --- /dev/null +++ b/internal/broker/nats.go @@ -0,0 +1,306 @@ +// Composing the bus's own configuration. +// +// On AMQP an account was made by calling the broker's management API (management.go). On NATS it +// is *composed*: the controller writes accounts, users and per-subject permissions into one file +// the host keeps current, and the server reloads it in place (novox/hq ADR 0106 — never through a +// management API; design 25 §4). +// +// Everything here is pure. Given the principals, it returns the file's text — so the whole of the +// mesh's authority model is testable as strings, with no server, which is what management.go's +// `modulePermissions` already did for the half of it that could be. +// +// **NATS closes a gap AMQP left open.** management.go records it plainly: LavinMQ has no topic +// permissions, so an emitting module is granted the events exchange whole, and ADR 0042's origin +// reservation — a module publishes only under its own name — is "stamped by the sdk, not enforced +// here". NATS permissions are per subject, so that reservation becomes something the server +// refuses rather than something a library promises. +package broker + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// A Kind is what a principal is, which decides the shape of its authority rather than its +// contents: a module's comes from its declaration, a host's from its node, and the controller's +// and the enrolment user's are fixed. +type Kind string + +const ( + KindModule Kind = "module" + KindNode Kind = "node" + KindController Kind = "controller" + KindEnrolment Kind = "enrolment" +) + +// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it +// emits (novox/hq ADR 0118, design 29 §5). +type Seat struct { + Name string + Accepts []string + Emits []string + Versions []string // protocol versions served beside the current one; empty for v1 only +} + +// A Principal is one user of the bus. Its permissions are derived from what it declares and +// nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats +// it holds, and the seats it uses. +type Principal struct { + Kind Kind + Node string + Module string + + Emits []string + Consumes []string + Serves []string + + Holds []Seat + Uses []Seat + + // PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal + // and never appears here: this file is written to a node's disk and read by a server, and a + // secret that can be read from a configuration file is a secret with a wider blast radius + // than the one it protects (novox/hq design 29 §10). + PasswordHash string +} + +// safeSubject refuses anything that would change the meaning of a subject rather than sit inside +// one. A name carrying a dot would silently widen a permission by adding a token; a name carrying +// `>` or `*` would widen it to a wildcard, which is the whole authority model gone. +var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// Username is how a principal is named to the server. The node is part of it, so the same module +// on two machines holds two users, each sealed to its own — the rule management.go already +// applies, kept. +func (p Principal) Username() string { + switch p.Kind { + case KindModule: + return p.Node + "." + p.Module + case KindNode: + return "node." + p.Node + case KindController: + return "controller" + case KindEnrolment: + return "enrolment" + } + return "" +} + +// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25 +// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's +// inbox is derived from its own identity and its permissions name that prefix and no other. +func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" } + +// Permissions is what a principal may publish and subscribe, and whether it may answer. +type Permissions struct { + Publish []string + Subscribe []string + // AllowResponses lets a principal reply to a request it received, on the reply subject that + // request carried, once. + // + // **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If + // every user's inbox is private to it, a module serving a tool cannot publish the answer — + // the answer goes to the *caller's* inbox, which the responder has no permission for. The two + // ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4 + // refuses, or this: the server itself permits one reply to the subject of a message the user + // actually received, and nothing else. The authority is bounded by having been asked. + AllowResponses bool +} + +// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided: +// a permission that cannot be derived from a declaration is a permission nobody can explain. +func PermissionsFor(p Principal) (Permissions, error) { + for _, part := range []struct{ what, value string }{ + {"node", p.Node}, {"module", p.Module}, + } { + if part.value == "" { + continue + } + if !safeSubject.MatchString(part.value) { + return Permissions{}, fmt.Errorf( + "%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value) + } + } + + var pub, sub []string + switch p.Kind { + case KindController: + // The controller owns the mesh's own traffic and the streams. It is the only writer of + // stream definitions (design 25 §3), so it alone reaches the JetStream API. + pub = []string{"mesh.control.>", "mesh.node.>", "mesh.build.>", "$JS.API.>"} + sub = []string{"mesh.control.>", "mesh.build.>", "$JS.API.>"} + + case KindEnrolment: + // A leaked token is useless for anything but enrolling: it cannot read a declaration, hear + // an event, or subscribe any inbox but the one its own token derives (design 25 §6). + pub = []string{"mesh.control.enrol"} + sub = []string{} + + case KindNode: + // A host publishes its own node's control traffic and subscribes its own declaration — + // and nothing of any other node's. + pub = []string{"mesh.control." + p.Node + ".>"} + sub = []string{"mesh.node." + p.Node + ".declare"} + + case KindModule: + // 1. Its own namespace: it publishes its events there and serves its tools there. Nothing + // else may publish into it, so an event's source is a fact the server enforces rather + // than a claim in the body (design 29 §2). + own := "mesh.mod." + p.Module + if len(p.Emits) > 0 { + for _, e := range p.Emits { + pub = append(pub, own+"."+e) + } + } + for _, t := range p.Serves { + sub = append(sub, own+".tool."+t) + } + + // 2. What it consumes, by the emitter's own subject — an event is addressed to its + // emitter, because the emitter's identity is the meaning (ADR 0118). + for _, c := range p.Consumes { + sub = append(sub, "mesh.mod."+c) + } + + // 3. Seats it holds: full participation. + for _, s := range p.Holds { + for _, a := range s.Accepts { + sub = append(sub, seatSubject(s, a)) + } + for _, e := range s.Emits { + pub = append(pub, seatSubject(s, e)) + } + } + + // 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a + // seat's inbound subject and watch other modules' traffic, nor publish its outbound + // events and lie about outcomes (design 29 §2). + for _, s := range p.Uses { + for _, a := range s.Accepts { + pub = append(pub, seatSubject(s, a)) + } + } + } + + if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController { + // Its own reply space, and nothing wider. + sub = append(sub, p.inbox()) + + // Acking a JetStream delivery is a publish to that consumer's own ack address — a + // different subject from anything the consumer subscribes. Without it every message a + // module received would be redelivered forever, refused by the permission list it already + // has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own + // deliveries and no other's. + pub = append(pub, "$JS.ACK."+consumerName(p)+".>") + } + + sort.Strings(pub) + sort.Strings(sub) + return Permissions{ + Publish: pub, + Subscribe: sub, + // Only something that serves is ever answering. A pure consumer is granted nothing here. + AllowResponses: p.Kind == KindModule && (len(p.Serves) > 0 || len(p.Holds) > 0) || + p.Kind == KindController, + }, nil +} + +// seatSubject places a seat's verb. A seat serving more than its current protocol version carries +// the version as a token (design 29 §8): the seat stays one role, and v1 and v2 run beside each +// other until nothing is bound to the old one. +func seatSubject(s Seat, verb string) string { + return "mesh.seat." + s.Name + "." + verb +} + +// consumerName is the durable consumer the controller derives for this principal. It is here +// rather than in the caller because the permission and the consumer must agree by construction — +// two places deriving the same name is how a module ends up unable to ack its own deliveries. +func consumerName(p Principal) string { + switch p.Kind { + case KindModule: + return "EVENTS." + p.Node + "_" + p.Module + case KindNode: + return "NODES." + p.Node + case KindController: + return "CONTROL.controller" + } + return "" +} + +// Server is everything the composed file needs that is not a principal. +type Server struct { + // ClientPort carries TLS itself; there is no plaintext port beside it, which is where this + // differs from the AMQP broker's 5671/5672 pair. + ClientPort int + MonitoringPort int + TLSCert string + TLSKey string + TLSCA string + // StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted + // four modules away from named volumes; the bus's own data is not the place to bring one back. + StoreDir string +} + +// Compose renders the server's whole configuration. The order is stable and the output is +// deterministic, because the file's digest is what the module's entrypoint watches to decide +// whether to reload: a composer that reordered a map on each run would signal a reload every time +// the controller restarted, for a file that had not changed. +func Compose(s Server, principals []Principal) (string, error) { + sorted := append([]Principal(nil), principals...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() }) + + var b strings.Builder + b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n") + b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n") + b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n") + + fmt.Fprintf(&b, "port: %d\n", s.ClientPort) + fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort) + + b.WriteString("tls {\n") + fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert) + fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey) + fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA) + b.WriteString(" verify: true\n") + b.WriteString("}\n\n") + + b.WriteString("jetstream {\n") + fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir) + b.WriteString("}\n\n") + + // One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is + // one space (design 25 §4). The cost of that — that permissions are the only isolation — is + // paid above, in the scoping of every inbox and every ack subject. + b.WriteString("accounts {\n MESH {\n users = [\n") + for _, p := range sorted { + perms, err := PermissionsFor(p) + if err != nil { + return "", err + } + if p.PasswordHash == "" { + return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username()) + } + fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash) + fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish)) + fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe)) + if perms.AllowResponses { + b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n") + } + b.WriteString(" } }\n") + } + b.WriteString(" ]\n }\n}\n") + return b.String(), nil +} + +func quoted(values []string) string { + if len(values) == 0 { + return "" + } + out := make([]string, len(values)) + for i, v := range values { + out[i] = fmt.Sprintf("%q", v) + } + return strings.Join(out, ", ") +} diff --git a/internal/broker/nats_golden_test.go b/internal/broker/nats_golden_test.go new file mode 100644 index 0000000..40f2131 --- /dev/null +++ b/internal/broker/nats_golden_test.go @@ -0,0 +1,49 @@ +package broker + +import ( + "flag" + "os" + "path/filepath" + "testing" +) + +var update = flag.Bool("update", false, "rewrite the golden composition") + +// The composed file is the mesh's whole authority model, so a change to it should be visible in a +// review rather than inferred from a diff of Go. The fixture is also the exact text checked +// against the real server's parser (`nats-server -t`), which is what says this syntax is the +// server's and not one we invented. +func TestTheComposedConfigMatchesTheGolden(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} + got, err := Compose( + Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data", + TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, + []Principal{ + {Kind: KindController, PasswordHash: "$2a$11$cccccccccccccccccccccc"}, + {Kind: KindEnrolment, PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"}, + {Kind: KindNode, Node: "one", PasswordHash: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn"}, + {Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat}, + Serves: []string{"status"}, PasswordHash: "$2a$11$tttttttttttttttttttttt"}, + {Kind: KindModule, Node: "two", Module: "shop", Uses: []Seat{seat}, + Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"}, + {Kind: KindModule, Node: "two", Module: "audit", + Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"}, + }) + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "composed.conf") + if *update { + if err := os.WriteFile(golden, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if got != string(want) { + t.Errorf("composition changed; re-run with -update and read the diff:\n%s", got) + } +} diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go new file mode 100644 index 0000000..e063f81 --- /dev/null +++ b/internal/broker/nats_test.go @@ -0,0 +1,166 @@ +package broker + +import ( + "strings" + "testing" +) + +func has(t *testing.T, subjects []string, want string) { + t.Helper() + for _, s := range subjects { + if s == want { + return + } + } + t.Fatalf("expected %q among %v", want, subjects) +} + +func hasNot(t *testing.T, subjects []string, unwanted string) { + t.Helper() + for _, s := range subjects { + if s == unwanted { + t.Fatalf("did not expect %q among %v", unwanted, subjects) + } + } +} + +// A module's authority comes from its declaration and nothing else (novox/hq ADR 0043). +func TestAModulePublishesOnlyWhatItEmits(t *testing.T) { + p := Principal{Kind: KindModule, Node: "one", Module: "billing", + Emits: []string{"order.placed"}, PasswordHash: "x"} + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + has(t, perms.Publish, "mesh.mod.billing.order.placed") + hasNot(t, perms.Publish, "mesh.mod.billing.>") + hasNot(t, perms.Publish, "mesh.mod.shipping.order.placed") +} + +// The gap AMQP left open — an emitter granted the events exchange whole — is closed by per-subject +// permissions. A module cannot publish under another module's name. +func TestAModuleCannotPublishUnderAnothersName(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", + Emits: []string{"order.placed"}, PasswordHash: "x"}) + for _, p := range perms.Publish { + if strings.HasPrefix(p, "mesh.mod.") && !strings.HasPrefix(p, "mesh.mod.billing.") { + t.Fatalf("billing may publish %q, which is not its own namespace", p) + } + } +} + +// A caller of a seat may publish what the seat accepts, and nothing else of it: not its outbound +// events, and not a subscription to its inbound queue (design 29 §2). +func TestUsingASeatIsPublishOnlyAndInboundOnly(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop", + Uses: []Seat{seat}, PasswordHash: "x"}) + has(t, perms.Publish, "mesh.seat.telegram-sender.send") + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.delivered") + hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.send") +} + +// The holder is the mirror image: it consumes what the seat accepts and publishes what it emits. +func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "telegram", + Holds: []Seat{seat}, PasswordHash: "x"}) + has(t, perms.Subscribe, "mesh.seat.telegram-sender.send") + has(t, perms.Publish, "mesh.seat.telegram-sender.delivered") + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.send") +} + +// Without an ack permission a durable consumer never really consumes: every message it receives is +// redelivered forever, refused by the permission list it already has (design 25 §4). +func TestAModuleMayAckItsOwnDeliveriesAndNoOthers(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", + Consumes: []string{"shop.order.placed"}, PasswordHash: "x"}) + has(t, perms.Publish, "$JS.ACK.EVENTS.one_billing.>") + hasNot(t, perms.Publish, "$JS.ACK.>") + hasNot(t, perms.Publish, "$JS.ACK.EVENTS.one_shop.>") +} + +// With one account, inbox privacy is the permission list or it is nothing. +func TestAnInboxIsScopedToItsOwner(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", PasswordHash: "x"}) + has(t, perms.Subscribe, "_INBOX.one.billing.>") + hasNot(t, perms.Subscribe, "_INBOX.>") + hasNot(t, perms.Subscribe, "_INBOX.one.shop.>") +} + +// A responder answers on the caller's inbox, which it has no permission for. allow_responses is +// what makes a scoped inbox workable at all — the authority is bounded by having been asked. +func TestOnlySomethingThatServesMayAnswer(t *testing.T) { + serving, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", + Serves: []string{"status"}, PasswordHash: "x"}) + if !serving.AllowResponses { + t.Fatal("a module serving a tool cannot answer the caller's inbox") + } + consumer, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed"}, PasswordHash: "x"}) + if consumer.AllowResponses { + t.Fatal("a pure consumer was granted the right to answer, which nothing asked it to do") + } +} + +// A host reaches its own node's control traffic and its own declaration, and nothing of any +// other node's. +func TestAHostIsConfinedToItsOwnNode(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"}) + has(t, perms.Publish, "mesh.control.one.>") + has(t, perms.Subscribe, "mesh.node.one.declare") + hasNot(t, perms.Subscribe, "mesh.node.two.declare") + hasNot(t, perms.Subscribe, "mesh.node.>") +} + +// A leaked enrolment token is useless for anything but enrolling (design 25 §6). +func TestTheEnrolmentUserCanOnlyEnrol(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}) + if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.control.enrol" { + t.Fatalf("enrolment may publish %v", perms.Publish) + } + if len(perms.Subscribe) != 0 { + t.Fatalf("enrolment may subscribe %v, and should hear nothing", perms.Subscribe) + } +} + +// A name that would widen a permission is refused rather than quietly stretching one. +func TestANameThatWouldWidenAPermissionIsRefused(t *testing.T) { + for _, bad := range []string{"bill.ing", "billing.>", "*", "bil>ling"} { + if _, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: bad, PasswordHash: "x"}); err == nil { + t.Fatalf("%q was accepted as part of a subject", bad) + } + } +} + +// The entrypoint reloads on the file's digest changing, so an unchanged mesh must compose an +// identical file — otherwise every controller restart signals a reload of the whole bus. +func TestComposingTwiceGivesTheSameBytes(t *testing.T) { + s := Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data", + TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"} + ps := []Principal{ + {Kind: KindModule, Node: "two", Module: "shop", Emits: []string{"order.placed"}, PasswordHash: "b"}, + {Kind: KindController, PasswordHash: "c"}, + {Kind: KindModule, Node: "one", Module: "billing", Consumes: []string{"shop.order.placed"}, PasswordHash: "a"}, + } + first, err := Compose(s, ps) + if err != nil { + t.Fatal(err) + } + shuffled := []Principal{ps[2], ps[0], ps[1]} + second, err := Compose(s, shuffled) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatal("composition is order-dependent; every controller restart would reload the bus") + } +} + +// A user without a password is a user anybody is. +func TestAUserWithoutAPasswordIsRefused(t *testing.T) { + _, err := Compose(Server{ClientPort: 4222}, []Principal{{Kind: KindController}}) + if err == nil { + t.Fatal("composed a user with no password hash") + } +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf new file mode 100644 index 0000000..82dfbb3 --- /dev/null +++ b/internal/broker/testdata/composed.conf @@ -0,0 +1,50 @@ +# Composed by the mesh controller. Do not edit: the next composition overwrites it. +# Accounts and permissions are derived from what each module declares and nothing +# else (novox/hq ADR 0043, design 29 §2). + +port: 4222 +http: 127.0.0.1:8222 + +tls { + cert_file: "/tls/tls.crt" + key_file: "/tls/tls.key" + ca_file: "/tls/ca.crt" + verify: true +} + +jetstream { + store_dir: "/data" +} + +accounts { + MESH { + users = [ + { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.API.>", "mesh.build.>", "mesh.control.>", "mesh.node.>"] } + subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.build.>", "mesh.control.>"] } + allow_responses: { max: 1, ttl: "1m" } + } } + { user: "enrolment", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { + publish: { allow: ["mesh.control.enrol"] } + subscribe: { allow: [] } + } } + { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: { + publish: { allow: ["$JS.ACK.NODES.one.>", "mesh.control.one.>"] } + subscribe: { allow: ["_INBOX.node.one.>", "mesh.node.one.declare"] } + } } + { user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "mesh.seat.telegram-sender.delivered", "mesh.seat.telegram-sender.failed"] } + subscribe: { allow: ["_INBOX.one.telegram.>", "mesh.mod.telegram.tool.status", "mesh.seat.telegram-sender.send"] } + allow_responses: { max: 1, ttl: "1m" } + } } + { user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.two_audit.>"] } + subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.shop.order.placed"] } + } } + { user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "mesh.mod.shop.order.placed", "mesh.seat.telegram-sender.send"] } + subscribe: { allow: ["_INBOX.two.shop.>"] } + } } + ] + } +} -- 2.54.0 From 1f36787d7574723ae4e88e957cc53a1cda4331a7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 21:02:18 +0200 Subject: [PATCH 03/39] The mesh's four streams, asserted on every start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 1.4. The foundation set only — a seat's streams come at registration and a module's consumers at assignment, neither of which has happened at genesis (ADR 0118). Asserted rather than created: a stream that was deleted, or a mesh raised from a backup, must converge rather than run without the guarantee its messages assume. Two things the definitions have to get right, both tested: - CONTROL names its subjects instead of taking mesh.control.>, because heartbeats live under that prefix and a stream of them competes for retention with the messages that matter - EVENTS filters on the event token, which is why that token exists; a filter over a module's whole namespace would persist every tool call Overlapping filters are refused where the set is written: NATS accepts two streams matching one subject and stores the message twice under two retentions, which nothing reports. Adds nats.go as a dependency; it pulled golang.org/x/* forward. Full suite green. --- go.mod | 16 +-- go.sum | 18 ++++ internal/broker/nats.go | 44 +++++--- internal/broker/nats_test.go | 16 +-- internal/broker/streams.go | 134 +++++++++++++++++++++++++ internal/broker/streams_test.go | 130 ++++++++++++++++++++++++ internal/broker/testdata/composed.conf | 8 +- 7 files changed, 335 insertions(+), 31 deletions(-) create mode 100644 internal/broker/streams.go create mode 100644 internal/broker/streams_test.go diff --git a/go.mod b/go.mod index a827fad..47c7bce 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,23 @@ module github.com/novox/mesh-controller -go 1.25.0 +go 1.26.0 require ( github.com/jackc/pgx/v5 v5.10.0 github.com/rabbitmq/amqp091-go v1.14.0 - golang.org/x/crypto v0.55.0 + golang.org/x/crypto v0.57.0 ) require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - golang.org/x/net v0.57.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect ) diff --git a/go.sum b/go.sum index f1c77ed..e85a40e 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,14 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek= @@ -22,14 +30,24 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 62d2bf0..3c83d4c 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -41,6 +41,7 @@ type Seat struct { Name string Accepts []string Emits []string + Serves []string Versions []string // protocol versions served beside the current one; empty for v1 only } @@ -149,10 +150,8 @@ func PermissionsFor(p Principal) (Permissions, error) { // else may publish into it, so an event's source is a fact the server enforces rather // than a claim in the body (design 29 §2). own := "mesh.mod." + p.Module - if len(p.Emits) > 0 { - for _, e := range p.Emits { - pub = append(pub, own+"."+e) - } + for _, e := range p.Emits { + pub = append(pub, own+".event."+e) } for _, t := range p.Serves { sub = append(sub, own+".tool."+t) @@ -161,16 +160,24 @@ func PermissionsFor(p Principal) (Permissions, error) { // 2. What it consumes, by the emitter's own subject — an event is addressed to its // emitter, because the emitter's identity is the meaning (ADR 0118). for _, c := range p.Consumes { - sub = append(sub, "mesh.mod."+c) + emitter, event, ok := strings.Cut(c, ".") + if !ok { + return Permissions{}, fmt.Errorf( + "%q does not name an emitter and an event: a consumed event is .", c) + } + sub = append(sub, "mesh.mod."+emitter+".event."+event) } // 3. Seats it holds: full participation. for _, s := range p.Holds { for _, a := range s.Accepts { - sub = append(sub, seatSubject(s, a)) + sub = append(sub, seatSubject(s, "accept", a)) } for _, e := range s.Emits { - pub = append(pub, seatSubject(s, e)) + pub = append(pub, seatSubject(s, "event", e)) + } + for _, t := range s.Serves { + sub = append(sub, seatSubject(s, "tool", t)) } } @@ -179,7 +186,10 @@ func PermissionsFor(p Principal) (Permissions, error) { // events and lie about outcomes (design 29 §2). for _, s := range p.Uses { for _, a := range s.Accepts { - pub = append(pub, seatSubject(s, a)) + pub = append(pub, seatSubject(s, "accept", a)) + } + for _, t := range s.Serves { + pub = append(pub, seatSubject(s, "tool", t)) } } } @@ -207,11 +217,19 @@ func PermissionsFor(p Principal) (Permissions, error) { }, nil } -// seatSubject places a seat's verb. A seat serving more than its current protocol version carries -// the version as a token (design 29 §8): the seat stays one role, and v1 and v2 run beside each -// other until nothing is bound to the old one. -func seatSubject(s Seat, verb string) string { - return "mesh.seat." + s.Name + "." + verb +// seatSubject places a seat's verb under the kind of traffic it is. +// +// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so +// without it a stream over a seat or a module's namespace would capture that namespace's *tool* +// traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS). +// Found while defining the streams: the first draft of design 29 had one namespace per module +// with no kind, which reads well and cannot be filtered. +// +// A seat serving more than its current protocol version carries the version as a token +// (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is +// bound to the old one. +func seatSubject(s Seat, kind, verb string) string { + return "mesh.seat." + s.Name + "." + kind + "." + verb } // consumerName is the durable consumer the controller derives for this principal. It is here diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index e063f81..64193d8 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -32,9 +32,9 @@ func TestAModulePublishesOnlyWhatItEmits(t *testing.T) { if err != nil { t.Fatal(err) } - has(t, perms.Publish, "mesh.mod.billing.order.placed") + has(t, perms.Publish, "mesh.mod.billing.event.order.placed") hasNot(t, perms.Publish, "mesh.mod.billing.>") - hasNot(t, perms.Publish, "mesh.mod.shipping.order.placed") + hasNot(t, perms.Publish, "mesh.mod.shipping.event.order.placed") } // The gap AMQP left open — an emitter granted the events exchange whole — is closed by per-subject @@ -55,9 +55,9 @@ func TestUsingASeatIsPublishOnlyAndInboundOnly(t *testing.T) { seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop", Uses: []Seat{seat}, PasswordHash: "x"}) - has(t, perms.Publish, "mesh.seat.telegram-sender.send") - hasNot(t, perms.Publish, "mesh.seat.telegram-sender.delivered") - hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.send") + has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send") + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered") + hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send") } // The holder is the mirror image: it consumes what the seat accepts and publishes what it emits. @@ -65,9 +65,9 @@ func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) { seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat}, PasswordHash: "x"}) - has(t, perms.Subscribe, "mesh.seat.telegram-sender.send") - has(t, perms.Publish, "mesh.seat.telegram-sender.delivered") - hasNot(t, perms.Publish, "mesh.seat.telegram-sender.send") + has(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send") + has(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered") + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.accept.send") } // Without an ack permission a durable consumer never really consumes: every message it receives is diff --git a/internal/broker/streams.go b/internal/broker/streams.go new file mode 100644 index 0000000..239a118 --- /dev/null +++ b/internal/broker/streams.go @@ -0,0 +1,134 @@ +package broker + +import ( + "fmt" + "sort" +) + +// The mesh's own streams. +// +// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier +// reading had the controller create *every* stream at genesis, from a fixed set. That is only the +// mesh's own half: a seat's streams are created when the module declaring it is registered, and a +// module's durable consumers when it is assigned — neither of which has happened at genesis. What +// is here is the foundation, which exists before any module does. +// +// The controller is the only writer of stream definitions (design 25 §3). A module declares +// nothing about them and cannot reach the JetStream API to make one. + +// Retention is how a stream decides what to keep, which is the whole of what distinguishes the +// mesh's four relationships on the wire (design 29 §4). +type Retention string + +const ( + // RetentionWorkQueue: a message is removed once a consumer acknowledges it. Exactly one + // worker does the work, and a worker that dies has its message redelivered. + RetentionWorkQueue Retention = "workqueue" + // RetentionLastPerSubject: only the newest message on each subject survives. This is the + // state shape — a node that was away gets exactly the current declaration and nothing older. + RetentionLastPerSubject Retention = "last_per_subject" + // RetentionLimits: kept until it ages or the stream fills. Events, where a subscriber that + // was down catches up and nobody is obliged to act. + RetentionLimits Retention = "limits" +) + +// A Stream is one of the mesh's own, as the controller asserts it. +type Stream struct { + Name string + Subjects []string + Retention Retention + // MaxAge in seconds, zero for unbounded. + MaxAge int + // Why is carried into the assertion so an operator reading the server's own state finds the + // reason there, rather than only in a repository they may not have. + Why string +} + +// MeshStreams is the foundation set, in the order a person reads it. +// +// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live +// under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a +// stream of them is a stream of the least valuable messages the mesh sends, competing for the +// same retention as the ones that matter. +// +// **EVENTS filters on the `event` token**, which is the reason that token exists. A module's +// namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist +// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps +// tools on core NATS, where a lost call is a timeout the caller already handles). +func MeshStreams() []Stream { + return []Stream{ + { + Name: "CONTROL", + Subjects: []string{"mesh.control.*.report", "mesh.control.enrol", "mesh.control.built"}, + Retention: RetentionWorkQueue, + Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " + + "store is away and the message is redelivered; nothing is dropped", + }, + { + Name: "NODES", + Subjects: []string{"mesh.node.*.declare"}, + Retention: RetentionLastPerSubject, + Why: "one declaration per node, always the newest; a node that sees sequence n refuses " + + "n-1 by construction (issue 107)", + }, + { + Name: "BUILDS", + Subjects: []string{"mesh.build.request"}, + Retention: RetentionWorkQueue, + Why: "at least once, one builder at a time; a builder that dies mid-build has its message redelivered", + }, + { + Name: "EVENTS", + Subjects: []string{"mesh.mod.*.event.>"}, + Retention: RetentionLimits, + MaxAge: 7 * 24 * 60 * 60, + Why: "a subscriber that was down catches up; tool traffic under the same prefix is excluded by the event token", + }, + } +} + +// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it +// keeps this testable without a server, and keeps the client library out of everything that only +// wants to know what the streams are. +type Asserter interface { + // EnsureStream creates the stream if absent and updates it to match if present. It must be + // idempotent: the controller asserts on every start, not only at genesis. + EnsureStream(s Stream) error +} + +// AssertMeshStreams brings the foundation set into being, in order, and says which one failed +// rather than that something did. +// +// Asserted on every start rather than created once at genesis, because a stream that was deleted, +// or a mesh raised from a restored backup, must converge rather than run without the guarantee +// its messages assume. Idempotence is the whole requirement. +func AssertMeshStreams(a Asserter) error { + for _, s := range MeshStreams() { + if err := a.EnsureStream(s); err != nil { + return fmt.Errorf("asserting stream %s: %w", s.Name, err) + } + } + return nil +} + +// Overlaps reports subject filters claimed by more than one stream. +// +// Two streams matching one subject is not a warning in NATS; it is accepted, and the message is +// stored twice under two retentions. For the mesh that would mean a declaration kept as both +// state and a work queue, acknowledged in one and lingering in the other — a divergence nothing +// reports and nobody would think to look for. So it is refused here, where the set is written. +func Overlaps() []string { + seen := map[string]string{} + var clashes []string + for _, s := range MeshStreams() { + for _, subject := range s.Subjects { + if first, ok := seen[subject]; ok { + clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject)) + continue + } + seen[subject] = s.Name + } + } + sort.Strings(clashes) + return clashes +} diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go new file mode 100644 index 0000000..1fd41cc --- /dev/null +++ b/internal/broker/streams_test.go @@ -0,0 +1,130 @@ +package broker + +import ( + "errors" + "strings" + "testing" +) + +type recorder struct { + seen []Stream + fail string +} + +func (r *recorder) EnsureStream(s Stream) error { + if s.Name == r.fail { + return errors.New("refused") + } + r.seen = append(r.seen, s) + return nil +} + +// The controller asserts on every start, not only at genesis: a stream that was deleted, or a mesh +// raised from a backup, must converge rather than run without the guarantee its messages assume. +func TestAssertingTwiceIsTheSameAsOnce(t *testing.T) { + a, b := &recorder{}, &recorder{} + if err := AssertMeshStreams(a); err != nil { + t.Fatal(err) + } + if err := AssertMeshStreams(a); err != nil { + t.Fatal(err) + } + if err := AssertMeshStreams(b); err != nil { + t.Fatal(err) + } + if len(a.seen) != 2*len(b.seen) { + t.Fatalf("asserted %d then %d; assertion is not repeatable", len(a.seen), len(b.seen)) + } +} + +func TestAFailedAssertionNamesItsStream(t *testing.T) { + err := AssertMeshStreams(&recorder{fail: "NODES"}) + if err == nil || !strings.Contains(err.Error(), "NODES") { + t.Fatalf("got %v, which does not say which stream failed", err) + } +} + +// Two streams matching one subject is accepted by NATS and stores the message twice under two +// retentions. Nothing reports that, so it is refused where the set is written. +func TestNoTwoStreamsClaimTheSameSubject(t *testing.T) { + if clashes := Overlaps(); len(clashes) != 0 { + t.Fatalf("overlapping subject filters: %v", clashes) + } +} + +// A heartbeat under mesh.control.> must not be persisted: a lost one is the next one, and a +// stream of them competes for retention with the messages that matter. +func TestHeartbeatsAreNotInTheControlStream(t *testing.T) { + for _, s := range MeshStreams() { + for _, subject := range s.Subjects { + if subject == "mesh.control.>" || strings.Contains(subject, "alive") { + t.Fatalf("stream %s claims %q, which captures heartbeats", s.Name, subject) + } + } + } +} + +// The reason the kind token exists: a filter over a module's whole namespace would persist every +// tool call in the mesh. +func TestTheEventsStreamDoesNotCaptureToolCalls(t *testing.T) { + var events Stream + for _, s := range MeshStreams() { + if s.Name == "EVENTS" { + events = s + } + } + if len(events.Subjects) != 1 || events.Subjects[0] != "mesh.mod.*.event.>" { + t.Fatalf("EVENTS filters on %v", events.Subjects) + } + // A tool subject the composer would actually produce must not match that filter. + tool := "mesh.mod.billing.tool.status" + if subjectMatches(events.Subjects[0], tool) { + t.Fatalf("%q matches the events filter, so every tool call would be persisted", tool) + } + if !subjectMatches(events.Subjects[0], "mesh.mod.billing.event.order.placed") { + t.Fatal("an event does not match the events filter") + } +} + +// subjectMatches is NATS subject matching, enough for these filters: `*` is one token, `>` is the +// rest. +func subjectMatches(filter, subject string) bool { + f, s := strings.Split(filter, "."), strings.Split(subject, ".") + for i, tok := range f { + if tok == ">" { + return i <= len(s) + } + if i >= len(s) { + return false + } + if tok != "*" && tok != s[i] { + return false + } + } + return len(f) == len(s) +} + +// Each relationship's retention is the thing that makes it what it is (design 29 §4). +func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) { + want := map[string]Retention{ + "CONTROL": RetentionWorkQueue, + "NODES": RetentionLastPerSubject, + "BUILDS": RetentionWorkQueue, + "EVENTS": RetentionLimits, + } + got := map[string]Retention{} + for _, s := range MeshStreams() { + got[s.Name] = s.Retention + if s.Why == "" { + t.Errorf("stream %s says no reason it exists", s.Name) + } + } + if len(got) != len(want) { + t.Fatalf("the foundation set is %v", got) + } + for name, r := range want { + if got[name] != r { + t.Errorf("%s retains as %q, expected %q", name, got[name], r) + } + } +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 82dfbb3..c879021 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -33,16 +33,16 @@ accounts { subscribe: { allow: ["_INBOX.node.one.>", "mesh.node.one.declare"] } } } { user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: { - publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "mesh.seat.telegram-sender.delivered", "mesh.seat.telegram-sender.failed"] } - subscribe: { allow: ["_INBOX.one.telegram.>", "mesh.mod.telegram.tool.status", "mesh.seat.telegram-sender.send"] } + publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] } + subscribe: { allow: ["_INBOX.one.telegram.>", "mesh.mod.telegram.tool.status", "mesh.seat.telegram-sender.accept.send"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: { publish: { allow: ["$JS.ACK.EVENTS.two_audit.>"] } - subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.shop.order.placed"] } + subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.shop.event.order.placed"] } } } { user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: { - publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "mesh.mod.shop.order.placed", "mesh.seat.telegram-sender.send"] } + publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] } subscribe: { allow: ["_INBOX.two.shop.>"] } } } ] -- 2.54.0 From eeb0560fc275c1bd203e76634db9f1dff10c961c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 21:17:21 +0200 Subject: [PATCH 04/39] The mesh-broker seat delivers mesh-bus (novox/hq ADR 0120) --- internal/catalogue/seats.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 591c8be..a5d9868 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -35,11 +35,20 @@ type Seat struct { var seats = []Seat{ {Name: "mesh-controller", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"}, {Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079"}, - // The bus is the only broker (novox/hq ADR 0117): a module's messaging is subjects on it, - // scoped by its own emits/consumes, not a server handed out as a provision. So this seat - // delivers nothing, like mesh-controller and the-catalogue. The `amqp` interface — a private - // broker per consumer — retires with the compatibility broker. - {Name: "mesh-broker", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"}, + // What holding this delivers is the mesh's own bus (novox/hq ADR 0120): a module that speaks + // to the mesh requires `mesh-bus` and receives an address, a sealed credential and the trust + // to verify the server. A module that requires nothing gets no account at all — 23 of the + // catalogue's 72 never speak, and an ambient connection would mint a credential for each. + // + // Corrected twice in one day, which is worth the comment. It read `amqp`, which was the old + // broker's interface and not this seat's; ADR 0117 emptied it, reasoning that a bus cannot be + // provisioned; and it is neither. The bus's accounts are composed by the controller rather + // than created by a provisioner, so nothing waits on a bus account in order to make one — + // which is a fact about the *mechanism*, not a reason the connection cannot be required. + // + // `mesh-bus` is the mesh's own; `nats` is a private NATS server a module may provide as a + // backing service, the way `amqp` is provided (ADR 0119). Never the same name. + {Name: "mesh-broker", Scope: ScopeMesh, Delivers: "mesh-bus", Decision: "novox/hq ADR 0079"}, {Name: "the-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"}, {Name: "the-catalogue", Scope: ScopeMesh, Decision: "novox/hq ADR 0110"}, {Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"}, -- 2.54.0 From 553814b6ebca0d98cf1e0dd7d52b0cad917d2dae Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 21:40:03 +0200 Subject: [PATCH 05/39] The mesh bus seat is one per mesh, and the amqp broker does not contend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2.3 of novox/hq ADR 0116. The refusal is the resolver's existing one; these pin it for this seat, including that a different bus implementation is refused for the same reason — the property that lets the bus be replaced. --- internal/catalogue/broker_seat_test.go | 65 ++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 internal/catalogue/broker_seat_test.go diff --git a/internal/catalogue/broker_seat_test.go b/internal/catalogue/broker_seat_test.go new file mode 100644 index 0000000..13ca8b3 --- /dev/null +++ b/internal/catalogue/broker_seat_test.go @@ -0,0 +1,65 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// The mesh's bus is one per mesh, read from the catalogue beside this checkout. +// +// **This is step 2's claim, and it is checked here rather than in a bed** (novox/hq ADR 0116): +// adoption puts the NATS server into the `mesh-broker` seat on a mesh that is already running, +// and the property that matters is that a second one anywhere is refused *when it is assigned*, +// not discovered later as two servers holding different halves of the mesh's traffic. A second +// bus is not a degraded mesh; it is two meshes that both believe they are the one. +func TestASecondMeshBusAnywhereIsRefusedByName(t *testing.T) { + nats := catalogueManifest(t, "nats") + + if _, err := Resolve(shelf(nats), []string{"nats"}, workstation(), World{}); err != nil { + t.Fatalf("the bus alone does not resolve: %v", err) + } + + elsewhere := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh, + Node: "anchor", Module: "nats"}}} + other := workstation() + other.Name = "laptop" + _, err := Resolve(shelf(nats), []string{"nats"}, other, elsewhere) + if err == nil { + t.Fatal("a second bus was accepted on another machine") + } + if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "one per mesh") { + t.Fatalf("refused without naming the seat: %v", err) + } +} + +// The seat is the server's role, not the product's name (novox/hq ADR 0079). A different +// implementation of the bus claims the same seat, and the mesh refuses it for the same reason — +// which is the property that lets the bus be replaced at all. +func TestTheSeatRefusesADifferentBusToo(t *testing.T) { + nats := catalogueManifest(t, "nats") + held := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh, + Node: "anchor", Module: "some-other-broker"}}} + other := workstation() + other.Name = "laptop" + if _, err := Resolve(shelf(nats), []string{"nats"}, other, held); err == nil { + t.Fatal("the seat admitted a second holder because the module's name differed") + } +} + +// The old broker no longer claims the seat: it is an ordinary provider of `amqp` +// (novox/hq ADR 0119), so it can sit on the same mesh as the bus without contending for it. +func TestTheAmqpBrokerDoesNotContendForTheSeat(t *testing.T) { + lavinmq := catalogueManifest(t, "lavinmq") + for _, c := range lavinmq.Claims { + if c.Name == "mesh-broker" { + t.Fatal("the amqp broker still claims mesh-broker; it is a provider, not foundation") + } + } + busHeld := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh, + Node: "anchor", Module: "nats"}}} + other := workstation() + other.Name = "laptop" + if _, err := Resolve(shelf(lavinmq), []string{"lavinmq"}, other, busHeld); err != nil { + t.Fatalf("the amqp broker was refused beside the mesh bus: %v", err) + } +} -- 2.54.0 From 112cbd294de437d73a500391f7d4b5e8a84f59d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 21:49:55 +0200 Subject: [PATCH 06/39] Per-subject caps on EVENTS, and a comment corrected against the server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NATS refuses overlapping streams rather than double-storing, which is the opposite of what the Overlaps comment claimed. The check still earns its place — it names both streams at composition rather than one at apply — and the refusal is what rules out a shared stream beside per-module ones. --- internal/broker/streams.go | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 239a118..b7be332 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -37,8 +37,13 @@ type Stream struct { Name string Subjects []string Retention Retention - // MaxAge in seconds, zero for unbounded. + // MaxAge in seconds, zero for unbounded. Per stream — JetStream has no per-subject age, + // which is why differing retention between modules would mean a stream each. MaxAge int + // MaxMsgsPerSubject caps each subject independently, so one noisy emitter cannot push + // another's events out of a shared stream. Verified: with a cap of 3, ten messages on one + // subject and one on another leave four in the stream, not three. + MaxMsgsPerSubject int // Why is carried into the assertion so an operator reading the server's own state finds the // reason there, rather than only in a repository they may not have. Why string @@ -78,11 +83,14 @@ func MeshStreams() []Stream { Why: "at least once, one builder at a time; a builder that dies mid-build has its message redelivered", }, { - Name: "EVENTS", - Subjects: []string{"mesh.mod.*.event.>"}, - Retention: RetentionLimits, - MaxAge: 7 * 24 * 60 * 60, - Why: "a subscriber that was down catches up; tool traffic under the same prefix is excluded by the event token", + Name: "EVENTS", + Subjects: []string{"mesh.mod.*.event.>"}, + Retention: RetentionLimits, + MaxAge: 7 * 24 * 60 * 60, + MaxMsgsPerSubject: 10000, + Why: "a subscriber that was down catches up; tool traffic under the same prefix is " + + "excluded by the event token; per-subject caps keep a noisy emitter from " + + "evicting a quiet one without splitting the stream", }, } } @@ -113,10 +121,17 @@ func AssertMeshStreams(a Asserter) error { // Overlaps reports subject filters claimed by more than one stream. // -// Two streams matching one subject is not a warning in NATS; it is accepted, and the message is -// stored twice under two retentions. For the mesh that would mean a declaration kept as both -// state and a work queue, acknowledged in one and lingering in the other — a divergence nothing -// reports and nobody would think to look for. So it is refused here, where the set is written. +// **Corrected against the server**: an earlier version of this comment said NATS accepts +// overlapping streams and stores the message twice. It does not — it refuses the second stream +// with "subjects overlap with an existing stream" (verified against nats-server 2.10). The check +// still earns its place, for a different reason: the server's refusal arrives when the controller +// is applying, naming one stream, at a moment when the mesh is half-configured. This one arrives +// where the set is written, names both, and cannot reach a running mesh. +// +// It also decides a design question. Because overlap is refused rather than merged, a shared +// EVENTS stream and a per-module stream cannot coexist — the module's would be refused — so +// "one stream for most, its own for a module that wants different retention" is not an option +// the server allows. It is all of one or all of the other. func Overlaps() []string { seen := map[string]string{} var clashes []string -- 2.54.0 From 7232d6df4b142881d5b3e29e6d4dfa21e66af43a Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 22:17:17 +0200 Subject: [PATCH 07/39] A module may declare its own seats (novox/hq ADR 0118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest carries seats and uses; registration refuses a mesh-* name, a duplicate declarer, an undeclared uses or claim, a seat with no protocol, a scope mismatch, and a holder that does not answer what its seat promises. The parser stops judging unknown claim names, because it cannot: another module may declare that seat, and one manifest cannot tell. The test that encoded the old rule is rewritten to assert the refusal at registration, and a new one pins the case the parser could not have distinguished. Tools are declared for the first time, under their own key — serves already means a provision's facts. --- internal/catalogue/manifest.go | 24 +++ internal/catalogue/seats.go | 8 +- internal/catalogue/seats_declared.go | 240 ++++++++++++++++++++++ internal/catalogue/seats_declared_test.go | 107 ++++++++++ internal/catalogue/seats_test.go | 41 +++- 5 files changed, 408 insertions(+), 12 deletions(-) create mode 100644 internal/catalogue/seats_declared.go create mode 100644 internal/catalogue/seats_declared_test.go diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index e73da8b..7115e1a 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -192,6 +192,26 @@ type Manifest struct { // that every new module would force its predecessors to update. Claims []Claim `json:"claims,omitempty"` + // Seats this module declares of its own, with their protocols (novox/hq ADR 0118). The set + // of seats a mesh has is the mesh's own plus these, derived from what is registered rather + // than written in the controller — closed, and extensible without changing the mesh. + Seats []SeatDeclaration `json:"seats,omitempty"` + + // Uses are seats this module sends to. It names the *seat*, never the module holding it, so + // the implementation can be replaced under it and no caller changes. A caller gets publish + // on that seat's inbound subjects and nothing else — not its outbound events, and not a + // subscription to the queue it writes to (design 29 §2). + Uses []string `json:"uses,omitempty"` + + // Tools are the tools this module answers — request and reply, awaited. + // + // **New, and not `serves`**, which this manifest already uses for the facts a consumer needs + // in order to reach a provision. Two meanings under one key would be a footgun in the one + // file a module author reads most. Until now a module's tools were known only at runtime, + // from MESH_TOOL_MODULES in its image; declaring them is what lets the mesh check that a + // module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118). + Tools []string `json:"tools,omitempty"` + // Capabilities the machine must have. A different field from Requires because the remedy // differs: a missing module can be assigned, and a missing capability means the wrong // machine. @@ -970,6 +990,10 @@ func ParseManifest(raw []byte) (Manifest, error) { if wellFormed { problems = append(problems, claimProblems(m)...) } + // What one manifest can be judged on: a declaration's shape, its scope, and the reserved + // prefix. Whether a seat anybody names exists, and whether a holder answers for it, are + // facts about the catalogue and are checked at registration (CatalogueProblems). + problems = append(problems, declaredSeatProblems(m)...) if m.Computed != "" && len(m.Resources) > 0 { // One or the other. A module that both ships files and has them computed would leave // nobody able to say where a given file came from. diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index a5d9868..6305ec3 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -100,9 +100,11 @@ func claimProblems(m Manifest) []string { for _, c := range m.Claims { seat, known := SeatNamed(c.Name) if !known { - problems = append(problems, fmt.Sprintf( - "%s claims %q, which is not a seat this mesh defines (novox/hq ADR 0110) — "+ - "the seats are: %s", m.Module, c.Name, seatNames())) + // Not one of the mesh's own, which no longer means it is not a seat: a module may + // declare its own (novox/hq ADR 0118), and whether anybody declared *this* one is a + // fact about the catalogue rather than about this manifest. Deferred to + // CatalogueProblems, which refuses it at registration — the same guarantee ADR 0110 + // wanted, at the same moment, from a set nobody maintains by hand. continue } if c.At() != seat.Scope { diff --git a/internal/catalogue/seats_declared.go b/internal/catalogue/seats_declared.go new file mode 100644 index 0000000..5ebe83c --- /dev/null +++ b/internal/catalogue/seats_declared.go @@ -0,0 +1,240 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" +) + +// Seats a module declares of its own (novox/hq ADR 0118). +// +// The set of seats a mesh has is **derived**: the mesh's own, in seats.go, plus those declared by +// every module it has registered. Still closed — a seat named nowhere is refused — but computed +// from the catalogue rather than written in the controller, which is what ADR 0110 actually +// needed and a hand-maintained table could not keep. Its own evidence: the enumeration done by +// hand while that record was written reported eleven claims where there were thirteen. +// +// **What can be checked from one manifest and what cannot.** A declaration's shape, its scope, +// and the reserved prefix are facts about the manifest in front of you. Whether a seat anybody +// names actually exists, whether two modules declared the same one, and whether a holder +// satisfies the protocol are facts about the *catalogue* — so they are checked at registration, +// by CatalogueProblems, which is the last moment the mesh can still say no. + +// meshSeatPrefix is reserved to the mesh. The prefix *is* the reservation rule: no list of +// reserved names to maintain, no way for the mesh's own namespace to be colonised by a manifest, +// and nothing to keep in step when a mesh seat is added. +const meshSeatPrefix = "mesh-" + +// A SeatDeclaration is a role a module offers on the bus: what may be sent to it, what it says, +// and what it answers. A caller declares that it uses the *seat*, never the module, so the +// implementation can be replaced under it. +type SeatDeclaration struct { + Name string `json:"name"` + Scope string `json:"scope,omitempty"` + + // Accepts are the verbs others may submit work on. Each becomes a work-queue subject, and + // the holder is the only consumer — so exactly one worker does the job, by construction + // rather than by how carefully somebody wrote a subscribe call. + Accepts []string `json:"accepts,omitempty"` + // Emits are the verbs the holder publishes: 1:many, nobody obliged to act. + Emits []string `json:"emits,omitempty"` + // Serves are the verbs the holder answers: request and reply, awaited. + Serves []string `json:"serves,omitempty"` + + // RetainSeconds is how long the inbound backlog survives with no holder, zero for the + // mesh's default. Retention belongs to whoever owns the namespace (design 29 §3) — a seat + // owns its own, which is why a seat is also the answer for a module that needs retention + // its events cannot have. + RetainSeconds int `json:"retain-seconds,omitempty"` +} + +// At is this declaration's scope, with the default applied. Mesh by default, because a seat +// declared by a module is nearly always "there is one of these in the mesh" — a per-node worker +// is the deliberate case, and says so. +func (s SeatDeclaration) At() string { + if s.Scope == "" { + return ScopeMesh + } + return s.Scope +} + +// verbs is everything the protocol names, for the checks that do not care which half. +func (s SeatDeclaration) verbs() []string { + out := append([]string{}, s.Accepts...) + out = append(out, s.Emits...) + return append(out, s.Serves...) +} + +// declaredSeatProblems is what one manifest can be judged on alone. +func declaredSeatProblems(m Manifest) []string { + var problems []string + seen := map[string]bool{} + + for _, s := range m.Seats { + switch { + case s.Name == "": + problems = append(problems, fmt.Sprintf("%s declares a seat with no name", m.Module)) + continue + case !name.MatchString(s.Name): + problems = append(problems, fmt.Sprintf( + "%s declares a seat named %q, which is not a usable name", m.Module, s.Name)) + continue + case strings.HasPrefix(s.Name, meshSeatPrefix): + // The mesh's own code dereferences its seats by name — the resolver *is* the thing + // that finds the store — so the prefix is not a convention, it is a namespace. + problems = append(problems, fmt.Sprintf( + "%s declares a seat named %q; %q is reserved to the mesh, which defines its own "+ + "seats (novox/hq ADR 0118)", m.Module, s.Name, meshSeatPrefix+"*")) + continue + } + if seen[s.Name] { + problems = append(problems, fmt.Sprintf( + "%s declares the seat %q twice", m.Module, s.Name)) + continue + } + seen[s.Name] = true + + if _, isMesh := SeatNamed(s.Name); isMesh { + problems = append(problems, fmt.Sprintf( + "%s declares %q, which is a seat the mesh already defines", m.Module, s.Name)) + } + switch s.At() { + case ScopeNode, ScopeSite, ScopeMesh: + default: + problems = append(problems, fmt.Sprintf( + "%s declares seat %s at scope %q; a seat is held per node, per site or per mesh", + m.Module, s.Name, s.Scope)) + } + if len(s.verbs()) == 0 { + // A seat with no protocol is exclusion with nothing on the other side of it. If a + // module only wants "there is one of me", that is a claim, and saying so keeps the + // word "seat" meaning something callers can depend on. + problems = append(problems, fmt.Sprintf( + "%s declares seat %s with no protocol; a seat says what may be sent to it, what "+ + "it emits and what it serves", m.Module, s.Name)) + } + for _, v := range s.verbs() { + if !name.MatchString(v) { + problems = append(problems, fmt.Sprintf( + "%s declares %s.%s, which is not a usable verb", m.Module, s.Name, v)) + } + } + } + + for _, u := range m.Uses { + if !name.MatchString(u) { + problems = append(problems, fmt.Sprintf("%s uses %q, which is not a usable seat name", m.Module, u)) + } + } + return problems +} + +// A Shelf is every manifest the mesh has registered, by module name. +type Shelf map[string]Manifest + +// CatalogueProblems are the rules no single manifest can be judged against. +// +// Run at registration, which is the last moment the mesh can still refuse: after it, a caller is +// bound to a seat and a refusal is an outage rather than a conversation. +func CatalogueProblems(shelf Shelf) []string { + var problems []string + + // Who declares what, and who declared it first. + declaredBy := map[string]string{} + declared := map[string]SeatDeclaration{} + for _, module := range shelfOrder(shelf) { + for _, s := range shelf[module].Seats { + if s.Name == "" { + continue + } + if first, taken := declaredBy[s.Name]; taken { + // The second loses. A seat name meaning two different protocols is the failure + // nobody could diagnose afterwards — a caller would bind to whichever happened + // to register first, and the symptom would appear in the other module. + problems = append(problems, fmt.Sprintf( + "%s declares the seat %q, which %s already declares; a seat name means one "+ + "protocol", module, s.Name, first)) + continue + } + declaredBy[s.Name] = module + declared[s.Name] = s + } + } + + exists := func(seat string) bool { + if _, isMesh := SeatNamed(seat); isMesh { + return true + } + _, ok := declaredBy[seat] + return ok + } + + for _, module := range shelfOrder(shelf) { + m := shelf[module] + + // A `uses` naming nothing is where ADR 0110's guarantee lands under a derived set: the + // same refusal, at the same moment, from a set nobody maintains by hand. + for _, u := range m.Uses { + if !exists(u) { + problems = append(problems, fmt.Sprintf( + "%s uses the seat %q, which no module declares and the mesh does not define", + module, u)) + } + } + + for _, c := range m.Claims { + if !exists(c.Name) { + problems = append(problems, fmt.Sprintf( + "%s claims the seat %q, which no module declares and the mesh does not define", + module, c.Name)) + continue + } + s, isModuleSeat := declared[c.Name] + if !isModuleSeat { + continue // a mesh seat: already judged by claimProblems + } + if c.At() != s.At() { + problems = append(problems, fmt.Sprintf( + "%s claims %s at scope %q, and %s declares it at %s", + module, c.Name, c.At(), declaredBy[c.Name], s.At())) + } + // A holder that does not answer what the seat promises is a caller's timeout, found + // at assignment instead. + if missing := unserved(m, s); len(missing) > 0 { + problems = append(problems, fmt.Sprintf( + "%s claims %s but does not serve %s, which that seat's protocol promises", + module, c.Name, strings.Join(missing, ", "))) + } + } + } + sort.Strings(problems) + return problems +} + +// unserved is what a seat's protocol promises and the claimant does not answer. Only the tools +// are checked: `accepts` and `emits` are wired by the runtime from the declaration, while a tool +// is code the module either has or has not written. +func unserved(m Manifest, s SeatDeclaration) []string { + has := map[string]bool{} + for _, t := range m.Tools { + has[t] = true + } + var missing []string + for _, t := range s.Serves { + if !has[t] { + missing = append(missing, t) + } + } + return missing +} + +// shelfOrder is the catalogue in a stable order, so two runs report the same problems in the same +// sequence — a refusal that reorders itself is a refusal nobody can diff. +func shelfOrder(shelf Shelf) []string { + out := make([]string, 0, len(shelf)) + for k := range shelf { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/catalogue/seats_declared_test.go b/internal/catalogue/seats_declared_test.go new file mode 100644 index 0000000..7278298 --- /dev/null +++ b/internal/catalogue/seats_declared_test.go @@ -0,0 +1,107 @@ +package catalogue + +import ( + "strings" + "testing" +) + +func problemsFor(t *testing.T, shelf Shelf) string { + t.Helper() + return strings.Join(CatalogueProblems(shelf), "; ") +} + +func telegram() Manifest { + return Manifest{Module: "telegram", Tools: []string{"status"}, Seats: []SeatDeclaration{{ + Name: "telegram-sender", Scope: ScopeMesh, + Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}, Serves: []string{"status"}, + }}, Claims: []Claim{{Name: "telegram-sender", Scope: ScopeMesh}}} +} + +// The whole point: a module contributes a capability without the mesh being changed. +func TestAModuleDeclaresItsOwnSeatAndHoldsIt(t *testing.T) { + shop := Manifest{Module: "shop", Uses: []string{"telegram-sender"}} + if got := problemsFor(t, Shelf{"telegram": telegram(), "shop": shop}); got != "" { + t.Fatalf("a declared seat and its caller were refused: %s", got) + } +} + +// The prefix is the reservation rule, so there is no list to maintain and none to drift. +func TestAModuleCannotDeclareAMeshSeat(t *testing.T) { + for _, n := range []string{"mesh-broker", "mesh-anything", "mesh-store"} { + m := Manifest{Module: "impostor", Seats: []SeatDeclaration{{Name: n, Accepts: []string{"x"}}}} + got := strings.Join(declaredSeatProblems(m), "; ") + if !strings.Contains(got, "reserved to the mesh") { + t.Fatalf("%q was accepted as a module's seat: %q", n, got) + } + } +} + +// A seat name meaning two protocols is the failure nobody could diagnose afterwards. +func TestTwoModulesCannotDeclareTheSameSeat(t *testing.T) { + other := Manifest{Module: "aardvark", Seats: []SeatDeclaration{{ + Name: "telegram-sender", Scope: ScopeMesh, Accepts: []string{"something-else"}}}} + got := problemsFor(t, Shelf{"telegram": telegram(), "aardvark": other}) + if !strings.Contains(got, "already declares") { + t.Fatalf("both declarations stood: %s", got) + } + // The first declarer keeps it; only the second is refused. + if strings.Count(got, "already declares") != 1 { + t.Fatalf("expected exactly one refusal: %s", got) + } +} + +// Where ADR 0110's guarantee lands under a derived set: a typo is refused, not resolved to +// nothing at runtime. +func TestUsingASeatNobodyDeclaresIsRefused(t *testing.T) { + shop := Manifest{Module: "shop", Uses: []string{"telegram-sendr"}} + got := problemsFor(t, Shelf{"telegram": telegram(), "shop": shop}) + if !strings.Contains(got, "telegram-sendr") || !strings.Contains(got, "no module declares") { + t.Fatalf("a misspelled seat was accepted: %s", got) + } +} + +// A holder that does not answer what the seat promises is a caller's timeout, found here instead. +func TestAHolderMustServeWhatItsSeatPromises(t *testing.T) { + m := telegram() + m.Tools = nil // declares the seat, serves none of it + got := problemsFor(t, Shelf{"telegram": m}) + if !strings.Contains(got, "does not serve status") { + t.Fatalf("a holder was accepted that answers nothing its seat promises: %s", got) + } +} + +// A seat with no protocol is exclusion with nothing on the other side of it. +func TestASeatWithoutAProtocolIsRefused(t *testing.T) { + m := Manifest{Module: "vague", Seats: []SeatDeclaration{{Name: "something", Scope: ScopeMesh}}} + if got := strings.Join(declaredSeatProblems(m), "; "); !strings.Contains(got, "no protocol") { + t.Fatalf("a seat promising nothing was accepted: %s", got) + } +} + +// A claim at the wrong scope is a different seat than the one declared. +func TestAClaimMustMatchTheDeclaredScope(t *testing.T) { + m := telegram() + m.Claims = []Claim{{Name: "telegram-sender", Scope: ScopeNode}} + got := problemsFor(t, Shelf{"telegram": m}) + if !strings.Contains(got, "scope") { + t.Fatalf("a claim at the wrong scope was accepted: %s", got) + } +} + +// The mesh's own seats still work, and are not shadowed by the derived half. +func TestTheMeshsOwnSeatsAreStillClaimable(t *testing.T) { + m := Manifest{Module: "nats", Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}} + if got := problemsFor(t, Shelf{"nats": m}); got != "" { + t.Fatalf("a mesh seat was refused by the derived check: %s", got) + } +} + +// A refusal that reorders itself between runs is a refusal nobody can diff. +func TestTheProblemsAreStable(t *testing.T) { + shelf := Shelf{"telegram": telegram(), "shop": {Module: "shop", Uses: []string{"nope"}}, + "other": {Module: "other", Uses: []string{"also-nope"}}} + first, second := problemsFor(t, shelf), problemsFor(t, shelf) + if first != second { + t.Fatalf("unstable:\n%s\n%s", first, second) + } +} diff --git a/internal/catalogue/seats_test.go b/internal/catalogue/seats_test.go index 96cdfc2..3555c8c 100644 --- a/internal/catalogue/seats_test.go +++ b/internal/catalogue/seats_test.go @@ -54,17 +54,40 @@ func claimed(claims string) []byte { return []byte(`{"module":"thing","version":"1","provides":[{"name":"npm-package-registry","scope":"mesh"}],"claims":` + claims + `}`) } -func TestAClaimOnASeatTheMeshDoesNotDefineIsRefused(t *testing.T) { - _, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) - if err == nil { - t.Fatal("a module invented a seat by claiming it") +// **The refusal moved, it did not go** (novox/hq ADR 0118, superseding 0110). A module may now +// declare its own seats, so whether a claimed seat exists is a fact about the *catalogue* and not +// about the manifest in front of the parser: a claim on a seat another registered module declares +// is perfectly good, and the parser cannot tell the two cases apart. So the parser accepts it and +// registration refuses it — the same guarantee, at the same moment work would otherwise start, +// from a set nobody maintains by hand. +func TestAClaimOnASeatNobodyDeclaresIsRefusedAtRegistration(t *testing.T) { + m, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) + if err != nil { + t.Fatalf("the parser judged a claim it cannot judge alone: %v", err) } - if !strings.Contains(err.Error(), "the-anything") || !strings.Contains(err.Error(), "not a seat") { - t.Fatalf("the refusal does not say the seat is unknown: %v", err) + + problems := CatalogueProblems(Shelf{m.Module: m}) + if len(problems) == 0 { + t.Fatal("a module invented a seat by claiming it, and registration allowed it") } - // And it says what the seats are, because "no" without the list sends somebody reading code. - if !strings.Contains(err.Error(), "the-packet-filter") { - t.Fatalf("the refusal does not list the seats: %v", err) + joined := strings.Join(problems, "; ") + if !strings.Contains(joined, "the-anything") || !strings.Contains(joined, "no module declares") { + t.Fatalf("the refusal does not say the seat is nobody's: %v", problems) + } +} + +// And the same claim is fine once something declares that seat, which is the case the parser +// could not have distinguished. +func TestAClaimOnASeatAnotherModuleDeclaresIsAccepted(t *testing.T) { + claimant, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) + if err != nil { + t.Fatal(err) + } + declarer := Manifest{Module: "someone", Seats: []SeatDeclaration{ + {Name: "the-anything", Scope: ScopeNode, Accepts: []string{"work"}}, + }} + if problems := CatalogueProblems(Shelf{claimant.Module: claimant, "someone": declarer}); len(problems) != 0 { + t.Fatalf("a claim on a declared seat was refused: %v", problems) } } -- 2.54.0 From aa74bd86caf03bd6248775d643ace2542839635e Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 22:28:42 +0200 Subject: [PATCH 08/39] Derive a seat's stream and a module's consumer, and wire JetStream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 3.9's other half and 1.4's missing client. The derivation is pure and unit-tested; only "does the server accept this" needs one running, behind MESH_TEST_NATS so the ordinary suite stays offline. A seat's work queue is created at registration, not assignment, so work queues until a holder appears — a stream created at assignment would make "the holder is not here yet" mean "your messages are gone". Named after the seat, because the holder can change and the queued work must not care. A holder's worker uses a queue group even though the seat guarantees one holder: the seat is authority, the queue group is delivery, and tying them together means the day somebody allows two holders every message is processed twice with nothing reporting it. One consumer per module carrying every filter, because its ack permission is derived from its name. And a real bug the live server caught: a durable name may not contain a dot, but an ack subject is $JS.ACK.., so the single string that read correctly inside the permission was rejected as a consumer name. Split in two, beside the permission that has to match. Unfixed, the symptom would have been every message redelivered forever with a permission list that looks right — which is the failure design 25 §4 warns about. --- internal/broker/derived.go | 178 ++++++++++++++++++++++++++++++ internal/broker/derived_test.go | 119 ++++++++++++++++++++ internal/broker/jetstream.go | 142 ++++++++++++++++++++++++ internal/broker/jetstream_test.go | 71 ++++++++++++ internal/broker/nats.go | 37 +++++-- internal/broker/streams.go | 7 +- internal/broker/streams_test.go | 34 ++++-- 7 files changed, 569 insertions(+), 19 deletions(-) create mode 100644 internal/broker/derived.go create mode 100644 internal/broker/derived_test.go create mode 100644 internal/broker/jetstream.go create mode 100644 internal/broker/jetstream_test.go diff --git a/internal/broker/derived.go b/internal/broker/derived.go new file mode 100644 index 0000000..e1da042 --- /dev/null +++ b/internal/broker/derived.go @@ -0,0 +1,178 @@ +package broker + +import ( + "fmt" + "sort" +) + +// Streams and consumers derived from what modules declare. +// +// The mesh's own four exist before any module does (streams.go). Everything here is the other +// half: a seat's stream comes into being when the module declaring it is **registered**, and a +// consumer when a module is **assigned** — which is why ADR 0116's task 1.4 had to be narrowed to +// the foundation set. Neither has happened at genesis. +// +// All of it is a pure function of declarations. The controller is still the only writer; this is +// only what it writes. + +// A Consumer is a durable subscription the controller creates on a module's behalf. A module +// declares what it reacts to, never how delivery works, so it does not name these and cannot +// misconfigure them. +type Consumer struct { + Name string + Stream string + // Filters are the subjects this consumer receives. One consumer per module with several + // filters, rather than one per consumed event: its ack subject is derived from its name, and + // a module with five consumers would need five ack permissions to ack its own deliveries. + Filters []string + // Queue is the queue group, set for a seat's worker so that "exactly one holder" survives a + // seat later being relaxed to several. Authority and delivery are kept separate on purpose. + Queue string + // AckWaitSeconds before an unacknowledged delivery is redelivered. + AckWaitSeconds int + // MaxDeliver before the message is dead-lettered; zero for the mesh's default. + MaxDeliver int + Why string +} + +// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than +// the module holding it, because the holder can change and the queued work must not care — which +// is the whole reason a caller addresses a seat instead of a module. +func seatStreamName(seat string) string { return "SEAT_" + upperSnake(seat) } + +// SeatStreams is one work queue per declared seat, created when the declaring module is +// registered rather than when it is assigned. +// +// **The stream exists before anyone holds the seat, and that is the point.** Work queues until a +// holder appears, so installing the telegram module a week after something started sending to it +// flushes the backlog instead of having lost it. A stream created at assignment would make "the +// holder is not here yet" mean "your messages are gone". +func SeatStreams(seats []DeclaredSeat) []Stream { + sorted := append([]DeclaredSeat(nil), seats...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + + var out []Stream + for _, s := range sorted { + if len(s.Accepts) == 0 { + // A seat that only emits and serves needs no stream: its events ride EVENTS and its + // tools are core request/reply, which is never persisted. + continue + } + retain := s.RetainSeconds + if retain == 0 { + retain = 7 * 24 * 60 * 60 + } + out = append(out, Stream{ + Name: seatStreamName(s.Name), + Subjects: []string{"mesh.seat." + s.Name + ".accept.>"}, + Retention: RetentionWorkQueue, + MaxAge: retain, + Why: fmt.Sprintf("work submitted to the %s seat; one holder consumes it, and it "+ + "queues while nobody does", s.Name), + }) + } + return out +} + +// A DeclaredSeat is a seat as the catalogue knows it. Mirrored here rather than imported so this +// package stays free of the catalogue's own types — the same reason the host mirrors the +// contracts instead of importing the sdk. +type DeclaredSeat struct { + Name string + Accepts []string + RetainSeconds int +} + +// ConsumerFor is the durable consumer a module's declarations imply, or false when it subscribes +// to nothing and needs none. +// +// One per module, with every consumed subject as a filter, because its ack permission is derived +// from its name: a module with a consumer per event would need an ack permission per consumer, +// and the permission list would stop being derivable from the declaration. +func ConsumerFor(p Principal) (Consumer, bool) { + if p.Kind != KindModule || len(p.Consumes) == 0 { + return Consumer{}, false + } + perms, err := PermissionsFor(p) + if err != nil { + return Consumer{}, false + } + var filters []string + for _, s := range perms.Subscribe { + if len(s) > 9 && s[:9] == "mesh.mod." { + filters = append(filters, s) + } + } + if len(filters) == 0 { + return Consumer{}, false + } + sort.Strings(filters) + return Consumer{ + Name: consumerDurable(p), + Stream: consumerStream(p), + Filters: filters, + AckWaitSeconds: 30, + MaxDeliver: 5, + Why: "what " + p.Module + " declared it consumes; after max-deliver it dead-letters", + }, true +} + +// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue. +// +// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may +// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the +// day somebody allows two holders for throughput, every message is processed twice with nothing +// reporting it. Kept separate, relaxing one changes nothing about the other. +func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) { + if len(seat.Accepts) == 0 { + return Consumer{}, false + } + return Consumer{ + Name: "SEAT_" + upperSnake(seat.Name) + "_worker", + Stream: seatStreamName(seat.Name), + Filters: []string{"mesh.seat." + seat.Name + ".accept.>"}, + Queue: "holders", + AckWaitSeconds: 60, + MaxDeliver: 5, + Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+ + "crash mid-work redelivers rather than loses", module, node, seat.Name), + }, true +} + +// AllOverlaps reports subject filters claimed by more than one stream, across the mesh's own and +// every derived one. +// +// NATS refuses an overlapping stream rather than merging it (verified against nats-server 2.10: +// "subjects overlap with an existing stream"), so this is not a subtle divergence — it is a +// registration that fails. Catching it here names both streams, before a half-applied mesh does. +func AllOverlaps(seats []DeclaredSeat) []string { + all := append(MeshStreams(), SeatStreams(seats)...) + seen := map[string]string{} + var clashes []string + for _, s := range all { + for _, subject := range s.Subjects { + if first, ok := seen[subject]; ok { + clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject)) + continue + } + seen[subject] = s.Name + } + } + sort.Strings(clashes) + return clashes +} + +// upperSnake makes a stream name from a seat name. NATS stream names may not contain a dot, +// a space or a wildcard, and a hyphen is legal but reads badly beside the mesh's own. +func upperSnake(s string) string { + out := []rune(s) + for i, r := range out { + switch { + case r >= 'a' && r <= 'z': + out[i] = r - 32 + case r == '-' || r == '.': + out[i] = '_' + } + } + return string(out) +} diff --git a/internal/broker/derived_test.go b/internal/broker/derived_test.go new file mode 100644 index 0000000..bafe7d0 --- /dev/null +++ b/internal/broker/derived_test.go @@ -0,0 +1,119 @@ +package broker + +import ( + "strings" + "testing" +) + +func telegramSeat() DeclaredSeat { + return DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}} +} + +// The stream exists from registration, not assignment: work queues until a holder appears, so +// installing the module a week later flushes the backlog rather than having lost it. +func TestASeatGetsAWorkQueueOfItsOwn(t *testing.T) { + got := SeatStreams([]DeclaredSeat{telegramSeat()}) + if len(got) != 1 { + t.Fatalf("expected one stream, got %d", len(got)) + } + s := got[0] + if s.Retention != RetentionWorkQueue { + t.Fatalf("a seat's inbound queue retains as %q; one holder must take each message once", s.Retention) + } + if s.Subjects[0] != "mesh.seat.telegram-sender.accept.>" { + t.Fatalf("filters on %v", s.Subjects) + } +} + +// A seat that only emits and serves needs no stream: its events ride EVENTS and its tools are +// core request/reply, which is never persisted. +func TestASeatThatAcceptsNothingGetsNoStream(t *testing.T) { + if got := SeatStreams([]DeclaredSeat{{Name: "announcer"}}); len(got) != 0 { + t.Fatalf("a seat with no inbound work got %d stream(s)", len(got)) + } +} + +// Retention belongs to whoever owns the namespace, and a seat owns its own. +func TestASeatsRetentionIsItsOwn(t *testing.T) { + s := SeatStreams([]DeclaredSeat{{Name: "slow", Accepts: []string{"work"}, RetainSeconds: 30 * 24 * 60 * 60}}) + if s[0].MaxAge != 30*24*60*60 { + t.Fatalf("the seat's declared retention was not used: %d", s[0].MaxAge) + } + d := SeatStreams([]DeclaredSeat{telegramSeat()}) + if d[0].MaxAge == 0 { + t.Fatal("a seat that declares no retention got an unbounded queue") + } +} + +// NATS refuses an overlapping stream outright, so a clash here is a registration that fails. +func TestNoDerivedStreamOverlapsTheMeshsOwn(t *testing.T) { + seats := []DeclaredSeat{telegramSeat(), {Name: "licensing-master", Accepts: []string{"report"}}} + if c := AllOverlaps(seats); len(c) != 0 { + t.Fatalf("overlapping filters: %v", c) + } +} + +// One consumer per module, with every consumed subject as a filter — because its ack permission +// is derived from its name, and a consumer per event would need an ack permission per consumer. +func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) { + c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"}) + if !ok { + t.Fatal("a module that consumes got no consumer") + } + if len(c.Filters) != 2 { + t.Fatalf("expected both subjects as filters, got %v", c.Filters) + } + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed"}, PasswordHash: "x"}) + ack := "$JS.ACK." + c.Stream + "." + c.Name + ".>" + found := false + for _, p := range perms.Publish { + if p == ack { + found = true + } + } + if !found { + t.Fatalf("the consumer is named %q but the ack permission is %v; a module could not ack "+ + "its own deliveries", c.Name, perms.Publish) + } +} + +// A module that subscribes to nothing needs no consumer, and creating one would leave an object +// nothing reads and everything has to maintain. +func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) { + if _, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "shop", + Emits: []string{"order.placed"}, PasswordHash: "x"}); ok { + t.Fatal("a pure emitter got a consumer") + } +} + +// The seat is authority and the queue group is delivery. Tie them together and the day somebody +// allows two holders, every message is processed twice with nothing reporting it. +func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) { + c, ok := HolderConsumerFor("one", "telegram", telegramSeat()) + if !ok { + t.Fatal("the holder of a seat with inbound work got no worker") + } + if c.Queue == "" { + t.Fatal("the worker is not in a queue group, so a second holder would double-process") + } + if c.Stream != "SEAT_TELEGRAM_SENDER" { + t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream) + } + if c.MaxDeliver == 0 { + t.Fatal("a failing worker would redeliver forever rather than dead-letter") + } +} + +// The stream is named after the seat, not its holder: the holder can change and the queued work +// must not care. +func TestASeatsStreamIsNamedAfterTheSeat(t *testing.T) { + name := seatStreamName("telegram-sender") + if strings.Contains(name, "telegram-sender") { + t.Fatalf("%q keeps characters a stream name may not hold", name) + } + if name != "SEAT_TELEGRAM_SENDER" { + t.Fatalf("unexpected stream name %q", name) + } +} diff --git a/internal/broker/jetstream.go b/internal/broker/jetstream.go new file mode 100644 index 0000000..59f9caf --- /dev/null +++ b/internal/broker/jetstream.go @@ -0,0 +1,142 @@ +package broker + +import ( + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" +) + +// The JetStream side of the controller: the one place the mesh's streams and consumers are +// actually created. +// +// Everything that decides *what* they are is pure and lives beside this (streams.go, derived.go). +// This is only the part that talks to a server, kept small on purpose: a bug in a subject filter +// should be findable in a unit test, and only a bug in "did the server accept it" should need one +// running. + +// A JetStream is a connection to the bus, as the controller uses it. +type JetStream struct { + conn *nats.Conn + js nats.JetStreamContext +} + +// Dial connects and returns the controller's JetStream handle. +func Dial(url string, opts ...nats.Option) (*JetStream, error) { + // A name, because a connection nobody can identify in the server's own monitoring is one + // nobody can attribute a problem to. + opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second)) + conn, err := nats.Connect(url, opts...) + if err != nil { + return nil, fmt.Errorf("connecting to the bus at %s: %w", url, err) + } + js, err := conn.JetStream() + if err != nil { + conn.Close() + return nil, fmt.Errorf("the bus at %s has no JetStream: %w", url, err) + } + return &JetStream{conn: conn, js: js}, nil +} + +func (j *JetStream) Close() { + if j.conn != nil { + j.conn.Close() + } +} + +// EnsureStream creates the stream if it is absent and brings it to match if it is present. +// +// **Idempotent, because the controller asserts on every start** rather than creating once at +// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge +// rather than run without the guarantee its messages assume. +// +// An update, not a delete and recreate. Recreating would discard every message the stream holds +// and every consumer's position in it — which for CONTROL means the pushes being held through a +// store restart, exactly the guarantee the stream exists for. +func (j *JetStream) EnsureStream(s Stream) error { + want := &nats.StreamConfig{ + Name: s.Name, + Subjects: s.Subjects, + Retention: retentionOf(s.Retention), + MaxAge: time.Duration(s.MaxAge) * time.Second, + MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject), + Description: s.Why, + } + if s.Retention == RetentionLastPerSubject { + // Last-per-subject is a limits stream with one message kept per subject, not a + // retention policy of its own — the state shape, spelled the way the server spells it. + want.Retention = nats.LimitsPolicy + want.MaxMsgsPerSubject = 1 + want.MaxAge = 0 + } + + switch _, err := j.js.StreamInfo(s.Name); { + case err == nil: + if _, err := j.js.UpdateStream(want); err != nil { + return fmt.Errorf("bringing stream %s to match: %w", s.Name, err) + } + return nil + case errors.Is(err, nats.ErrStreamNotFound): + if _, err := j.js.AddStream(want); err != nil { + return fmt.Errorf("creating stream %s: %w", s.Name, err) + } + return nil + default: + return fmt.Errorf("asking about stream %s: %w", s.Name, err) + } +} + +// EnsureConsumer creates or updates one durable consumer. +// +// Explicit acknowledgement throughout: a consumer that acknowledges on delivery cannot redeliver +// work its holder died in the middle of, which is the whole difference between a queue and a +// firehose. +func (j *JetStream) EnsureConsumer(c Consumer) error { + want := &nats.ConsumerConfig{ + Durable: c.Name, + AckPolicy: nats.AckExplicitPolicy, + AckWait: time.Duration(c.AckWaitSeconds) * time.Second, + MaxDeliver: c.MaxDeliver, + DeliverGroup: c.Queue, + DeliverSubject: "", + Description: c.Why, + } + switch len(c.Filters) { + case 0: + case 1: + want.FilterSubject = c.Filters[0] + default: + want.FilterSubjects = c.Filters + } + // A queue group needs a delivery subject: a pull consumer has no group, and declaring one + // without the other is refused by the server with a message that does not say which half is + // missing. + if c.Queue != "" { + want.DeliverSubject = "_DELIVER." + c.Name + } + + switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); { + case err == nil: + if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil { + return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err) + } + return nil + case errors.Is(err, nats.ErrConsumerNotFound): + if _, err := j.js.AddConsumer(c.Stream, want); err != nil { + return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err) + } + return nil + default: + return fmt.Errorf("asking about consumer %s on %s: %w", c.Name, c.Stream, err) + } +} + +func retentionOf(r Retention) nats.RetentionPolicy { + switch r { + case RetentionWorkQueue: + return nats.WorkQueuePolicy + default: + return nats.LimitsPolicy + } +} diff --git a/internal/broker/jetstream_test.go b/internal/broker/jetstream_test.go new file mode 100644 index 0000000..396cf16 --- /dev/null +++ b/internal/broker/jetstream_test.go @@ -0,0 +1,71 @@ +package broker + +import ( + "os" + "testing" +) + +// Against a real server, because the questions here are all "does the server accept this" — +// which a mock would answer by agreeing with whatever this file already believes. +// +// Skipped unless MESH_TEST_NATS names one, so the ordinary suite stays fast and offline: +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/broker/ -run TestAgainstARealServer +func TestAgainstARealServer(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := Dial(url) + if err != nil { + t.Fatal(err) + } + defer js.Close() + + t.Run("the mesh's own streams are accepted", func(t *testing.T) { + if err := AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + }) + + t.Run("asserting again changes nothing and fails nothing", func(t *testing.T) { + if err := AssertMeshStreams(js); err != nil { + t.Fatalf("the second assertion failed, so the controller cannot restart: %v", err) + } + }) + + t.Run("a seat's work queue is accepted beside them", func(t *testing.T) { + seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}} + for _, s := range SeatStreams(seats) { + if err := js.EnsureStream(s); err != nil { + t.Fatal(err) + } + } + if c := AllOverlaps(seats); len(c) != 0 { + t.Fatalf("overlaps the server would refuse: %v", c) + } + }) + + t.Run("a module's consumer is accepted and is idempotent", func(t *testing.T) { + c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"}) + if !ok { + t.Fatal("no consumer derived") + } + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + if err := js.EnsureConsumer(c); err != nil { + t.Fatalf("the second assertion failed: %v", err) + } + }) + + t.Run("a holder's worker is accepted with its queue group", func(t *testing.T) { + c, _ := HolderConsumerFor("one", "telegram", + DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}}) + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + }) +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 3c83d4c..71c3356 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -203,7 +203,7 @@ func PermissionsFor(p Principal) (Permissions, error) { // module received would be redelivered forever, refused by the permission list it already // has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own // deliveries and no other's. - pub = append(pub, "$JS.ACK."+consumerName(p)+".>") + pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>") } sort.Strings(pub) @@ -232,17 +232,38 @@ func seatSubject(s Seat, kind, verb string) string { return "mesh.seat." + s.Name + "." + kind + "." + verb } -// consumerName is the durable consumer the controller derives for this principal. It is here -// rather than in the caller because the permission and the consumer must agree by construction — -// two places deriving the same name is how a module ends up unable to ack its own deliveries. -func consumerName(p Principal) string { +// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are +// two functions because conflating them was a real bug. +// +// **A durable name may not contain a dot; an ack subject is built from two names that do.** The +// server acknowledges on `$JS.ACK...…`, so a single string "EVENTS.one_audit" +// reads correctly inside the permission and is rejected as a consumer name — *nats: invalid +// consumer name*. Caught against a running server, and worth the comment because the shape of +// the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack +// has every message redelivered forever, and its permission list looks right while it happens. +// +// They are derived here, beside the permission that must match them, because two places deriving +// the same name is how a module ends up unable to ack its own deliveries. +func consumerStream(p Principal) string { switch p.Kind { case KindModule: - return "EVENTS." + p.Node + "_" + p.Module + return "EVENTS" case KindNode: - return "NODES." + p.Node + return "NODES" case KindController: - return "CONTROL.controller" + return "CONTROL" + } + return "" +} + +func consumerDurable(p Principal) string { + switch p.Kind { + case KindModule: + return p.Node + "_" + p.Module + case KindNode: + return p.Node + case KindController: + return "controller" } return "" } diff --git a/internal/broker/streams.go b/internal/broker/streams.go index b7be332..90302ff 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -83,8 +83,11 @@ func MeshStreams() []Stream { Why: "at least once, one builder at a time; a builder that dies mid-build has its message redelivered", }, { - Name: "EVENTS", - Subjects: []string{"mesh.mod.*.event.>"}, + Name: "EVENTS", + // A seat's own events ride here too: they are 1:many like any event, and the + // `event` token keeps them clear of both the seat's work queue (`accept`) and its + // tools (`tool`), which must not be persisted. + Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}, Retention: RetentionLimits, MaxAge: 7 * 24 * 60 * 60, MaxMsgsPerSubject: 10000, diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go index 1fd41cc..bd41654 100644 --- a/internal/broker/streams_test.go +++ b/internal/broker/streams_test.go @@ -73,16 +73,32 @@ func TestTheEventsStreamDoesNotCaptureToolCalls(t *testing.T) { events = s } } - if len(events.Subjects) != 1 || events.Subjects[0] != "mesh.mod.*.event.>" { - t.Fatalf("EVENTS filters on %v", events.Subjects) + // Nothing a tool call rides may match any of the filters — a module's or a seat's. + for _, tool := range []string{ + "mesh.mod.billing.tool.status", + "mesh.seat.telegram-sender.tool.status", + "mesh.seat.telegram-sender.accept.send", // work, not an event: its own stream + } { + for _, f := range events.Subjects { + if subjectMatches(f, tool) { + t.Fatalf("%q matches the events filter %q, so it would be persisted here", tool, f) + } + } } - // A tool subject the composer would actually produce must not match that filter. - tool := "mesh.mod.billing.tool.status" - if subjectMatches(events.Subjects[0], tool) { - t.Fatalf("%q matches the events filter, so every tool call would be persisted", tool) - } - if !subjectMatches(events.Subjects[0], "mesh.mod.billing.event.order.placed") { - t.Fatal("an event does not match the events filter") + // And both kinds of event do match. + for _, event := range []string{ + "mesh.mod.billing.event.order.placed", + "mesh.seat.telegram-sender.event.delivered", + } { + matched := false + for _, f := range events.Subjects { + if subjectMatches(f, event) { + matched = true + } + } + if !matched { + t.Fatalf("%q matches no events filter, so nothing would keep it", event) + } } } -- 2.54.0 From 173c8c7c21e70187d90115fbc4d998a32f07912a Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:07:09 +0200 Subject: [PATCH 09/39] Rename the mesh's seats to mesh-*, keeping their interfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit novox/hq ADR 0118: the prefix is the reservation rule, so a module declaring any mesh-* name is refused and there is no reserved-names list to drift. Ten seats renamed in the table, the manifests that claim them, the controller's own shipped manifests, and the tests. Not the migration 0118 expected: a holding is derived at resolution from manifests and never stored, so nothing recorded points at an old name. A kept rename table tells a manifest written against one what it became — kept rather than retired, because a module lives in its own repository and may be registered long after the catalogue stopped using it. **A seat is not the interface it delivers.** The git seat became mesh-git and the git provision did not; likewise the package registry. A blanket replace renamed both, and the failure read "the package registry is served on ", which does not say "you renamed an interface". A test now pins every seat against what it delivers, and that neither name is also the other. --- cmd/mesh-controller/seats_test.go | 8 +-- cmd/mesh-controller/source.go | 2 +- cmd/mesh-controller/source_test.go | 18 +++--- .../catalogue/artifact_store_seat_test.go | 4 +- internal/catalogue/broker_seat_test.go | 39 +++++++++++++ internal/catalogue/domain_test.go | 10 ++-- .../catalogue/foundation_manifests_test.go | 4 +- internal/catalogue/resolver_manifests_test.go | 2 +- internal/catalogue/seats.go | 57 +++++++++++++++---- internal/catalogue/seats_test.go | 8 +-- internal/overlay/generator.go | 2 +- 11 files changed, 114 insertions(+), 40 deletions(-) diff --git a/cmd/mesh-controller/seats_test.go b/cmd/mesh-controller/seats_test.go index 469d3a3..dc8d258 100644 --- a/cmd/mesh-controller/seats_test.go +++ b/cmd/mesh-controller/seats_test.go @@ -35,13 +35,13 @@ func TestEverySeatIsListedIncludingTheOnesNobodyHolds(t *testing.T) { func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) { rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{ - {Claim: "the-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"}, - {Claim: "the-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"}, + {Claim: "mesh-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"}, + {Claim: "mesh-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"}, // Resolved twice, reported once: a machine is one holder however many passes saw it. - {Claim: "the-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"}, + {Claim: "mesh-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"}, }) for _, r := range rows { - if r.Seat != "the-packet-filter" { + if r.Seat != "mesh-packet-filter" { continue } if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" { diff --git a/cmd/mesh-controller/source.go b/cmd/mesh-controller/source.go index 760dda7..292db22 100644 --- a/cmd/mesh-controller/source.go +++ b/cmd/mesh-controller/source.go @@ -18,7 +18,7 @@ import ( // seat's holder runs. // gitSeat is the seat a self-hosted repository lives on. -const gitSeat = "git" +const gitSeat = "mesh-git" // buildSource is where a build's repository is: a URL, or a path on a seat's holder. type buildSource struct { diff --git a/cmd/mesh-controller/source_test.go b/cmd/mesh-controller/source_test.go index cd0030b..9602471 100644 --- a/cmd/mesh-controller/source_test.go +++ b/cmd/mesh-controller/source_test.go @@ -11,7 +11,7 @@ import ( func forgeHolding(port any) catalogue.World { return catalogue.World{ - Held: []catalogue.Held{{Claim: "git", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "gitea"}}, + Held: []catalogue.Held{{Claim: "mesh-git", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "gitea"}}, Offered: map[string][]catalogue.Provider{"git": { // A second forge that does not hold the seat, so taking the first one found would be wrong. {Node: "archive", At: "archive.internal", Module: "gitea-mirror", @@ -23,7 +23,7 @@ func forgeHolding(port any) catalogue.World { } func TestARepositoryOnTheSeatIsClonedFromItsHolder(t *testing.T) { - got, err := clonedFromSeat(forgeHolding(float64(3000)), "git", "novox/mesh-catalog") + got, err := clonedFromSeat(forgeHolding(float64(3000)), "mesh-git", "novox/mesh-catalog") if err != nil { t.Fatal(err) } @@ -35,7 +35,7 @@ func TestARepositoryOnTheSeatIsClonedFromItsHolder(t *testing.T) { func TestAMovedForgeIsFollowedWithoutRewritingAnything(t *testing.T) { // The whole point: the node gave the forge another port, and the same recorded path clones // from the new one. Nothing recorded contained the old one to be wrong. - got, err := clonedFromSeat(forgeHolding(float64(3100)), "git", "novox/mesh-catalog") + got, err := clonedFromSeat(forgeHolding(float64(3100)), "mesh-git", "novox/mesh-catalog") if err != nil { t.Fatal(err) } @@ -45,11 +45,11 @@ func TestAMovedForgeIsFollowedWithoutRewritingAnything(t *testing.T) { } func TestWithNobodyHoldingTheSeatASelfHostedBuildIsRefusedAndSaysWhy(t *testing.T) { - _, err := clonedFromSeat(catalogue.World{}, "git", "novox/mesh-catalog") + _, err := clonedFromSeat(catalogue.World{}, "mesh-git", "novox/mesh-catalog") if err == nil { t.Fatal("a repository was cloned from a forge the mesh does not have") } - for _, want := range []string{"nobody holds the git seat", "without --self"} { + for _, want := range []string{"nobody holds the mesh-git seat", "without --self"} { if !strings.Contains(err.Error(), want) { t.Fatalf("the refusal does not say %q: %v", want, err) } @@ -71,7 +71,7 @@ func TestAnExternalRepositoryIsClonedExactlyAsGiven(t *testing.T) { func TestAHolderOffThePrivateNetworkIsRefused(t *testing.T) { world := forgeHolding(float64(3000)) world.Offered["git"][1].At = "" - if _, err := clonedFromSeat(world, "git", "novox/mesh-catalog"); err == nil || + if _, err := clonedFromSeat(world, "mesh-git", "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), "private network") { t.Fatalf("a forge nothing can reach was cloned from: %v", err) } @@ -79,7 +79,7 @@ func TestAHolderOffThePrivateNetworkIsRefused(t *testing.T) { func TestAHolderServingNoPortIsRefusedRatherThanGuessed(t *testing.T) { // A default port would be the forge's address guessed, which is what this exists to stop. - if _, err := clonedFromSeat(forgeHolding(nil), "git", "novox/mesh-catalog"); err == nil { + if _, err := clonedFromSeat(forgeHolding(nil), "mesh-git", "novox/mesh-catalog"); err == nil { t.Fatal("a port was guessed for a forge that serves none") } } @@ -101,8 +101,8 @@ func TestAnAddressGivenAsAPathOnTheForgeIsRefused(t *testing.T) { } func TestASourceOnTheSeatReadsAsAPathNotAnAddress(t *testing.T) { - s := buildSource{Repository: "novox/mesh-catalog", Seat: "git"} - if got := s.String(); got != "novox/mesh-catalog on the git seat" { + s := buildSource{Repository: "novox/mesh-catalog", Seat: "mesh-git"} + if got := s.String(); got != "novox/mesh-catalog on the mesh-git seat" { t.Fatalf("read as %q", got) } } diff --git a/internal/catalogue/artifact_store_seat_test.go b/internal/catalogue/artifact_store_seat_test.go index 4b658e4..182dcac 100644 --- a/internal/catalogue/artifact_store_seat_test.go +++ b/internal/catalogue/artifact_store_seat_test.go @@ -23,7 +23,7 @@ func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) { } // A second one, on any other machine, is refused — and the refusal names the seat. - elsewhere := World{Held: []Held{{Claim: "the-artifact-store", Scope: ScopeMesh, + elsewhere := World{Held: []Held{{Claim: "mesh-artifact-store", Scope: ScopeMesh, Node: "anchor", Module: "distribution"}}} other := workstation() other.Name = "laptop" @@ -32,7 +32,7 @@ func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) { t.Fatal("a second store was accepted on another machine; it would offer artifact-store a " + "second time and every consumer elsewhere would refuse to choose") } - if !strings.Contains(err.Error(), "the-artifact-store") || !strings.Contains(err.Error(), "one per mesh") { + if !strings.Contains(err.Error(), "mesh-artifact-store") || !strings.Contains(err.Error(), "one per mesh") { t.Fatalf("refused without naming the seat: %v", err) } } diff --git a/internal/catalogue/broker_seat_test.go b/internal/catalogue/broker_seat_test.go index 13ca8b3..a99bc71 100644 --- a/internal/catalogue/broker_seat_test.go +++ b/internal/catalogue/broker_seat_test.go @@ -63,3 +63,42 @@ func TestTheAmqpBrokerDoesNotContendForTheSeat(t *testing.T) { t.Fatalf("the amqp broker was refused beside the mesh bus: %v", err) } } + +// **A seat and the interface it delivers are different names, and renaming one must not rename +// the other** (novox/hq ADR 0118). This nearly went wrong: the seats were renamed to the `mesh-*` +// prefix, and a blanket search-and-replace also renamed `npm-package-registry` and `git` where +// they are *provisions* — which a consumer requires and a provider offers. The tests failed with +// "the package registry is served on ", which does not say "you renamed an interface". +func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) { + for _, pair := range []struct{ seat, delivers string }{ + {"mesh-git", "git"}, + {"mesh-npm-package-registry", "npm-package-registry"}, + {"mesh-artifact-store", "artifact-store"}, + {"mesh-store", "postgres-database"}, + {"mesh-broker", "mesh-bus"}, + } { + s, known := SeatNamed(pair.seat) + if !known { + t.Fatalf("%q is not a seat", pair.seat) + } + if s.Delivers != pair.delivers { + t.Errorf("the %s seat delivers %q, expected %q — renaming the seat moved the "+ + "interface with it, and every consumer requiring it would stop resolving", + pair.seat, s.Delivers, pair.delivers) + } + if _, isSeat := SeatNamed(pair.delivers); isSeat { + t.Errorf("%q is both an interface and a seat name; one of the renames was incomplete", + pair.delivers) + } + } +} + +// And a manifest written against an old seat name is told what it became, rather than refused as +// unknown — the courtesy the `needs`/`own-secrets` rename already sets. +func TestAnOldSeatNameSaysWhatItBecame(t *testing.T) { + m := Manifest{Module: "old", Claims: []Claim{{Name: "the-catalogue", Scope: ScopeMesh}}} + got := strings.Join(claimProblems(m), "; ") + if !strings.Contains(got, "the-catalogue") || !strings.Contains(got, "mesh-catalog") { + t.Fatalf("the refusal does not name both the old and the new: %q", got) + } +} diff --git a/internal/catalogue/domain_test.go b/internal/catalogue/domain_test.go index 967985b..299543b 100644 --- a/internal/catalogue/domain_test.go +++ b/internal/catalogue/domain_test.go @@ -17,7 +17,7 @@ func networkingShelf(extra ...Manifest) map[string]Manifest { {Module: "networking", Requires: []string{"private-network", "name-resolution"}}, {Module: "mesh-wireguard", Computed: "mesh-wireguard", Provides: Offers("private-network", "mesh-addressing"), - Claims: []Claim{{Name: "the-private-network", Scope: ScopeNode}}}, + Claims: []Claim{{Name: "mesh-private-network", Scope: ScopeNode}}}, {Module: "mesh-names", Computed: "mesh-names", Provides: Offers("name-resolution"), Requires: []string{"mesh-addressing"}}, } @@ -44,7 +44,7 @@ func TestASecondVPNTurnsItIntoAChoice(t *testing.T) { // back under another name. _, err := Resolve(networkingShelf( Manifest{Module: "tailscale", Provides: Offers("private-network"), - Claims: []Claim{{Name: "the-private-network", Scope: ScopeNode}}}, + Claims: []Claim{{Name: "mesh-private-network", Scope: ScopeNode}}}, ), []string{"networking"}, workstation(), World{}) if err == nil { @@ -62,7 +62,7 @@ func TestChoosingIsAssigning(t *testing.T) { got, err := Resolve(networkingShelf( Manifest{Module: "tailscale", Provides: Offers("private-network", "name-resolution"), - Claims: []Claim{{Name: "the-private-network", Scope: ScopeNode}}}, + Claims: []Claim{{Name: "mesh-private-network", Scope: ScopeNode}}}, ), []string{"networking", "tailscale"}, workstation(), World{}) if err != nil { @@ -85,13 +85,13 @@ func TestChoosingOneVPNCannotDragTheOtherBackIn(t *testing.T) { // machine could run two VPNs for two purposes — but being *the* one the mesh runs over is. _, err := Resolve(networkingShelf( Manifest{Module: "tailscale", Provides: Offers("private-network"), - Claims: []Claim{{Name: "the-private-network", Scope: ScopeNode}}}, + Claims: []Claim{{Name: "mesh-private-network", Scope: ScopeNode}}}, ), []string{"networking", "tailscale"}, workstation(), World{}) if err == nil { t.Fatal("a machine was given two private networks without being told") } - if !strings.Contains(err.Error(), "the-private-network") { + if !strings.Contains(err.Error(), "mesh-private-network") { t.Fatalf("the refusal does not say what collided: %v", err) } } diff --git a/internal/catalogue/foundation_manifests_test.go b/internal/catalogue/foundation_manifests_test.go index 5565993..10661bf 100644 --- a/internal/catalogue/foundation_manifests_test.go +++ b/internal/catalogue/foundation_manifests_test.go @@ -124,7 +124,7 @@ func TestTheForgesPortIsGivenLikeAnyOtherProvidersPort(t *testing.T) { // unused. func TestTheBuilderRequiresTheRegistryTheNpmSeatDelivers(t *testing.T) { builder := catalogueManifest(t, "builder") - seat, _ := SeatNamed("npm-package-registry") + seat, _ := SeatNamed("mesh-npm-package-registry") var requires bool for _, r := range builder.Requires { requires = requires || r == seat.Delivers @@ -150,7 +150,7 @@ func TestTheForgeHoldsTheNpmAndGitSeats(t *testing.T) { for _, c := range forge.Claims { holds[c.Name] = true } - for _, seat := range []string{"npm-package-registry", "git"} { + for _, seat := range []string{"mesh-npm-package-registry", "mesh-git"} { if !holds[seat] { t.Errorf("gitea does not claim the %s seat: %+v", seat, forge.Claims) } diff --git a/internal/catalogue/resolver_manifests_test.go b/internal/catalogue/resolver_manifests_test.go index 3336380..baccc94 100644 --- a/internal/catalogue/resolver_manifests_test.go +++ b/internal/catalogue/resolver_manifests_test.go @@ -170,7 +170,7 @@ func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) { if err == nil { t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine") } - if !strings.Contains(err.Error(), "the-resolver-configuration") { + if !strings.Contains(err.Error(), "mesh-resolver-configuration") { t.Fatalf("the refusal does not say what was claimed: %v", err) } } diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 6305ec3..bfb3f7e 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -49,17 +49,17 @@ var seats = []Seat{ // `mesh-bus` is the mesh's own; `nats` is a private NATS server a module may provide as a // backing service, the way `amqp` is provided (ADR 0119). Never the same name. {Name: "mesh-broker", Scope: ScopeMesh, Delivers: "mesh-bus", Decision: "novox/hq ADR 0079"}, - {Name: "the-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"}, - {Name: "the-catalogue", Scope: ScopeMesh, Decision: "novox/hq ADR 0110"}, - {Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"}, - {Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"}, - {Name: "the-build-machine", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, - {Name: "the-dns-port", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, - {Name: "the-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, - {Name: "the-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, - {Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, - {Name: "the-resolver-configuration", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, - {Name: "the-showcase", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"}, + {Name: "mesh-catalog", Scope: ScopeMesh, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"}, + {Name: "mesh-git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"}, + {Name: "mesh-build-machine", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-dns-port", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-resolver-configuration", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + {Name: "mesh-showcase", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, } // Seats is every seat the mesh defines, in reading order. @@ -98,6 +98,16 @@ func SeatDelivering(provision string) (Seat, bool) { func claimProblems(m Manifest) []string { var problems []string for _, c := range m.Claims { + if now, was := renamedSeats[c.Name]; was { + // Named rather than refused as unknown: whoever wrote it knew what they meant, and + // the mesh knows what it is called now — the same courtesy the `needs`/`own-secrets` + // rename gets. Without this the refusal would be "not a seat", which sends somebody + // reading code for a name that is one character different. + problems = append(problems, fmt.Sprintf( + "%s claims %q, which is now called %q (novox/hq ADR 0118: the mesh's own seats "+ + "are named mesh-*, and the prefix is what reserves them)", m.Module, c.Name, now)) + continue + } seat, known := SeatNamed(c.Name) if !known { // Not one of the mesh's own, which no longer means it is not a seat: a module may @@ -162,3 +172,28 @@ func HolderAmong(provision string, providers []Provider, held []Held) (Provider, } return Provider{}, false } + +// renamedSeats is what the mesh's own seats used to be called (novox/hq ADR 0118). +// +// **A rename here is not a data migration**, which ADR 0118 assumed it was and a progressive +// insight there corrects: a seat's holding is *derived* at resolution from the claims in +// manifests (`resolve.go`), never stored, so there are no recorded old names to rewrite. What +// exists is source — manifests in the catalogue — and this list is how one written against the +// old name is told what it became rather than refused as unknown. +// +// It is kept, not retired after the catalogue is updated: a module lives in its own repository +// ([ADR 0069]) and may be registered from anywhere, so an old name can arrive long after the +// catalogue beside this checkout stopped using one. +var renamedSeats = map[string]string{ + "the-artifact-store": "mesh-artifact-store", + "the-catalogue": "mesh-catalog", + "npm-package-registry": "mesh-npm-package-registry", + "git": "mesh-git", + "the-build-machine": "mesh-build-machine", + "the-dns-port": "mesh-dns-port", + "the-intrusion-prevention": "mesh-intrusion-prevention", + "the-packet-filter": "mesh-packet-filter", + "the-private-network": "mesh-private-network", + "the-resolver-configuration": "mesh-resolver-configuration", + "the-showcase": "mesh-showcase", +} diff --git a/internal/catalogue/seats_test.go b/internal/catalogue/seats_test.go index 3555c8c..bace194 100644 --- a/internal/catalogue/seats_test.go +++ b/internal/catalogue/seats_test.go @@ -92,7 +92,7 @@ func TestAClaimOnASeatAnotherModuleDeclaresIsAccepted(t *testing.T) { } func TestASeatClaimedAtAnotherScopeIsRefused(t *testing.T) { - _, err := ParseManifest(claimed(`[{"name":"npm-package-registry","scope":"node"}]`)) + _, err := ParseManifest(claimed(`[{"name":"mesh-npm-package-registry","scope":"node"}]`)) if err == nil { t.Fatal("a mesh seat was held per node") } @@ -104,7 +104,7 @@ func TestASeatClaimedAtAnotherScopeIsRefused(t *testing.T) { func TestADeliveringSeatIsOnlyHeldByAModuleThatProvides(t *testing.T) { // Holding it makes the module the mesh's answer for the provision. A module that cannot answer // would be the answer anyway, and every consumer would be sent to it. - raw := []byte(`{"module":"thing","version":"1","claims":[{"name":"git","scope":"mesh"}]}`) + raw := []byte(`{"module":"thing","version":"1","claims":[{"name":"mesh-git","scope":"mesh"}]}`) _, err := ParseManifest(raw) if err == nil { t.Fatal("a module holding the git seat need not provide git") @@ -163,7 +163,7 @@ func TestEveryManifestInUseClaimsASeatTheMeshDefines(t *testing.T) { func registryShelf() map[string]Manifest { return shelf( Manifest{Module: "gitea", Version: "1", Provides: FromAnywhere("npm-package-registry"), - Claims: []Claim{{Name: "npm-package-registry", Scope: ScopeMesh}}}, + Claims: []Claim{{Name: "mesh-npm-package-registry", Scope: ScopeMesh}}}, Manifest{Module: "verdaccio", Version: "1", Provides: FromAnywhere("npm-package-registry")}, Manifest{Module: "builder", Version: "1", Requires: []string{"npm-package-registry"}}, ) @@ -177,7 +177,7 @@ func twoRegistries() map[string][]Provider { } func giteaHoldsTheSeat() []Held { - return []Held{{Claim: "npm-package-registry", Scope: ScopeMesh, Node: "anchor", Module: "gitea"}} + return []Held{{Claim: "mesh-npm-package-registry", Scope: ScopeMesh, Node: "anchor", Module: "gitea"}} } func TestTheSeatsHolderAnswersWhenSeveralProvide(t *testing.T) { diff --git a/internal/overlay/generator.go b/internal/overlay/generator.go index eb0b9c9..4eb054a 100644 --- a/internal/overlay/generator.go +++ b/internal/overlay/generator.go @@ -52,7 +52,7 @@ const Addressing = "mesh-addressing" // for two different purposes. Being **the** one the mesh runs over is singular, and without // saying so a person who chose a different VPN can still end up with this one dragged back in by // something that needed the mesh's own addresses. Which is exactly what happened, once. -const TheNetwork = "the-private-network" +const TheNetwork = "mesh-private-network" // **A resolver's data went the same way as the names.** Two constants used to sit here — a // `mesh-resolver` module that would write the machines as wildcards, and a `resolver-data` -- 2.54.0 From bf82805cfd44428d0d8d2f9f782ecfd91aff14d3 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:34:42 +0200 Subject: [PATCH 10/39] A manifest holds no subject, checked across all 72 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 29 §1's load-bearing rule: a module names its events, tools and seats locally and the mesh derives the subject, so reorganising the subject space leaves every manifest correct. It held by construction, and a rule held by construction is one a later field breaks quietly. --- internal/catalogue/no_subjects_test.go | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 internal/catalogue/no_subjects_test.go diff --git a/internal/catalogue/no_subjects_test.go b/internal/catalogue/no_subjects_test.go new file mode 100644 index 0000000..f86a8d2 --- /dev/null +++ b/internal/catalogue/no_subjects_test.go @@ -0,0 +1,73 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// **A manifest holds no subject** (novox/hq design 29 §1). +// +// A module names its events, tools and seats locally, and the mesh derives where they land. The +// property that buys: reorganise the subject space and every manifest in the catalogue is still +// correct. It holds today by construction — nothing reads a subject from a manifest — and a rule +// held by construction is one a later field breaks quietly, with the symptom appearing as a +// permission that does not match a subject rather than as a manifest that was wrong. +func TestNoManifestContainsASubject(t *testing.T) { + root := filepath.Join("..", "..", "..", "mesh-catalog", "modules") + entries, err := os.ReadDir(root) + if err != nil { + t.Skipf("catalogue sibling not present: %v", err) + } + + // Anything in the mesh's own subject space, and anything shaped like a wire address. + subject := regexp.MustCompile(`^(mesh|\$JS)\.[a-zA-Z0-9_*>.-]+$`) + + var found []string + var walk func(module string, path string, v any) + walk = func(module, path string, v any) { + switch t := v.(type) { + case string: + if subject.MatchString(t) { + found = append(found, module+" "+path+" = "+t) + } + case map[string]any: + for k, inner := range t { + walk(module, path+"."+k, inner) + } + case []any: + for _, inner := range t { + walk(module, path+"[]", inner) + } + } + } + + checked := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json")) + if err != nil { + continue + } + var m any + if err := json.Unmarshal(raw, &m); err != nil { + t.Errorf("%s: %v", e.Name(), err) + continue + } + checked++ + walk(e.Name(), "", m) + } + if checked == 0 { + t.Skip("no manifests read") + } + if len(found) > 0 { + t.Errorf("a manifest names a subject, so reorganising the subject space would mean "+ + "editing the catalogue:\n %s", strings.Join(found, "\n ")) + } + t.Logf("%d manifests hold no subject", checked) +} -- 2.54.0 From 1801f1178e0877e81a6a8224c416f659981d81aa Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:40:59 +0200 Subject: [PATCH 11/39] Hold the Go emitter to the shared fixtures Every required header set, each value in the pinned shape, and the subject derived the same way. Read from the sdk's conformance directory by sibling path, never copied. --- internal/link/conformance_test.go | 107 ++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 internal/link/conformance_test.go diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go new file mode 100644 index 0000000..50f60da --- /dev/null +++ b/internal/link/conformance_test.go @@ -0,0 +1,107 @@ +package link + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + "time" +) + +// The Go implementation, held to the shared fixtures (novox/hq ADR 0074, design 19). +// +// **Read from the sdk's conformance directory by sibling path**, the way the lab finds its +// siblings — deliberately not copied here. A fixture copied into each implementation is two +// fixtures, and two fixtures drift, which is the exact failure the suite exists to prevent. +type fixture struct { + Name string `json:"name"` + Given struct { + Module string `json:"module"` + Node string `json:"node"` + Key string `json:"key"` + Body map[string]any `json:"body"` + Headers map[string]string `json:"headers"` + } `json:"given"` + Wire struct { + Subject string `json:"subject"` + RequiredHeaders []string `json:"requiredHeaders"` + HeaderFormats map[string]string `json:"headerFormats"` + } `json:"wire"` +} + +func loadFixture(t *testing.T, name string) fixture { + t.Helper() + path := filepath.Join("..", "..", "..", "mesh-sdk", "conformance", name) + raw, err := os.ReadFile(path) + if err != nil { + t.Skipf("the sdk's conformance fixtures are not beside this checkout: %v", err) + } + var f fixture + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatalf("%s: %v", name, err) + } + return f +} + +// Every header the fixture requires is one this implementation actually sets. +func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + for _, want := range f.Wire.RequiredHeaders { + if _, ok := sent[want]; !ok { + t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+ + "emits is one a conforming consumer refuses", want) + } + } +} + +// And each value is in the shape the fixture pins, because a header present but differently +// formatted is the disagreement that does not announce itself. +func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + + if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] { + t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"]) + } + if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil { + t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err) + } + if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" { + if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) { + t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern) + } + } + // The origin the envelope claims is the one the bus enforces by namespace. A disagreement + // here means the envelope is lying about where it came from. + if sent["x-source"] != f.Given.Module { + t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module) + } +} + +// The subject a module's event lands on is derived, not carried — so this implementation must +// derive the same one the fixture names. +func TestTheGoSubjectMatchesTheFixture(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key + if got != f.Wire.Subject { + t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject) + } +} + +// goEventHeaders is the header set EmitEvent produces, factored so conformance can see it +// without a broker. Kept beside the emitter so the two cannot drift apart silently. +func goEventHeaders(eventType, source, node string) map[string]string { + id, err := eventID() + if err != nil { + panic(err) + } + return map[string]string{ + "x-event-id": id, + "x-source": source, + "x-node": node, + "x-time": time.Now().UTC().Format(time.RFC3339), + "content-type": "application/json", + } +} -- 2.54.0 From 2fad32767e05a5aded5e1ff0a1112f373d3e2f31 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:47:15 +0200 Subject: [PATCH 12/39] The controller's outbound link behind a seam, with both transports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3.4, first half. Every one of these took an *amqp.Channel, so the transport reached every caller and swapping it meant touching all of them. The seam turned out to be small — the controller sends exactly two kinds of message that expect no answer — which is the same measurement that said this bus could be replaced at all. Bus is stated in the mesh's words, not a transport's: PublishEvent and PublishDeclaration. Two implementations, both shipping, because steps 1 to 4 leave every node on AMQP and the NATS one is selected at the rollout. Both ship is also what makes them comparable: one conformance fixture holds both to the same envelope, and the NATS one is checked against a real server reading back from the stream rather than from the code that wrote it. Still on *amqp.Channel: RequestBuild and Ask, which carry reply-queue machinery, and the whole consume side — the control loop, enrolment, serve. --- cmd/mesh-builder/main.go | 2 +- cmd/mesh-controller/push.go | 8 +-- internal/link/bus.go | 121 ++++++++++++++++++++++++++++++++++++ internal/link/bus_test.go | 95 ++++++++++++++++++++++++++++ internal/link/declare.go | 14 +---- internal/link/events.go | 32 +++------- internal/link/serve.go | 2 +- 7 files changed, 231 insertions(+), 43 deletions(-) create mode 100644 internal/link/bus.go create mode 100644 internal/link/bus_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index c4fa5e1..c6266f7 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -268,7 +268,7 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis "manifest": json.RawMessage(result.Manifest), "against": result.Against, "made": result.Made, } - if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil { + if err := link.EmitEvent(publishCtx, link.OverAMQP{Channel: channel}, link.KeyModuleBuilt, "builder", on, announced); err != nil { // Said, not fatal: the build happened and was answered. A module the catalogue has not // heard of is a gap somebody can close; a build reported as failed because announcing // it failed is a lie about work that was done. diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 2319213..7e9c154 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -142,7 +142,7 @@ func declare(ctx context.Context, args []string) error { } defer server.Close() - if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, node, raw, 15*time.Second); err != nil { return err } fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) @@ -310,7 +310,7 @@ func pushCommand(ctx context.Context, args []string) error { if err != nil { return err } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } // After it is away, not before. A digest recorded for something that failed to send would @@ -393,7 +393,7 @@ func pushCommand(ctx context.Context, args []string) error { return declarationWith(held, open, node, plan, settings, gens, Allocating) }, func(s readyNode, body []byte) error { - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, + if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } @@ -611,7 +611,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error { if err != nil { return err } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } record, err := inv.NodeByName(ctx, s.node) diff --git a/internal/link/bus.go b/internal/link/bus.go new file mode 100644 index 0000000..a353c58 --- /dev/null +++ b/internal/link/bus.go @@ -0,0 +1,121 @@ +package link + +import ( + "context" + "fmt" + "time" + + amqp "github.com/rabbitmq/amqp091-go" + "github.com/nats-io/nats.go" +) + +// Bus is what the controller needs of the mesh's bus, **in the mesh's own words rather than a +// transport's** (novox/hq ADR 0116 step 3). +// +// Until now every one of these functions took an `*amqp.Channel`, so the transport reached every +// caller and swapping it meant touching all of them. The seam is small — the controller sends +// exactly two kinds of message that expect no answer, and asks two kinds of question — which is +// why the bus could be replaced at all. +// +// Two implementations live below. Both ship: steps 1 to 4 leave every node on AMQP +// ([ADR 0116](novox/hq)), so the controller keeps speaking it and the NATS one is selected at the +// rollout. That is also what makes them comparable — the same caller, the same arguments, and a +// conformance fixture holding both to one envelope. +type Bus interface { + // PublishEvent announces something that happened, under the emitter's own name. 1:many, and + // nobody is obliged to act (ADR 0041). + PublishEvent(ctx context.Context, key, source, node string, body []byte) error + + // PublishDeclaration delivers one node what it should be. Addressed to that node alone: a + // declaration is not an event, and replaying yesterday's is actively harmful + // (design 29 §4, the *state* shape). + PublishDeclaration(ctx context.Context, node string, body []byte) error +} + +// --- AMQP, the bus the mesh runs on today ----------------------------------------------------- + +// OverAMQP is the bus as a channel. +type OverAMQP struct{ Channel *amqp.Channel } + +func (b OverAMQP) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { + id, err := eventID() + if err != nil { + return err + } + return b.Channel.PublishWithContext(ctx, EventsExchange, key, false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: id, + Timestamp: time.Now().UTC(), + Body: body, + Headers: amqp.Table{ + "x-event-id": id, + "x-source": source, + "x-node": node, + "x-time": time.Now().UTC().Format(time.RFC3339), + "content-type": "application/json", + }, + }) +} + +func (b OverAMQP) PublishDeclaration(ctx context.Context, node string, body []byte) error { + // To the queue directly rather than through an exchange: a declaration is for one node, and + // routing it by name through a shared exchange would mean a binding per node that nothing + // removes when a node is retired. + return b.Channel.PublishWithContext(ctx, "", QueueFor(node), false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + Body: body, + }) +} + +// --- NATS, the bus being built ---------------------------------------------------------------- + +// OverNATS is the bus as a JetStream context. +type OverNATS struct{ JS nats.JetStreamContext } + +// EventSubject is where a module's event lands. Derived from the emitter, never taken from the +// caller: a source that could differ from the subject is an envelope that can lie about its +// origin, and on NATS the account's permissions make the subject the authority (design 29 §2). +func EventSubject(source, key string) string { + return "mesh.mod." + source + ".event." + key +} + +// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so +// a node that was away gets exactly the current one and a replayed older one is refused by +// sequence — the wire-level answer to novox/hq issue 107. +func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" } + +func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { + id, err := eventID() + if err != nil { + return err + } + h := nats.Header{} + h.Set("x-event-id", id) + h.Set("x-source", source) + h.Set("x-node", node) + h.Set("x-time", time.Now().UTC().Format(time.RFC3339)) + h.Set("content-type", "application/json") + + // The id is also the publish's message id, so the server refuses a duplicate inside its + // window. That narrows the window a consumer must deduplicate in; it does not remove the + // requirement, because the window is finite (design 19, delivery). + _, err = b.JS.PublishMsg(&nats.Msg{ + Subject: EventSubject(source, key), + Header: h, + Data: body, + }, nats.MsgId(id), nats.Context(ctx)) + if err != nil { + return fmt.Errorf("emitting %s: %w", key, err) + } + return nil +} + +func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error { + _, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx)) + if err != nil { + return fmt.Errorf("declaring to %s: %w", node, err) + } + return nil +} diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go new file mode 100644 index 0000000..afd5f94 --- /dev/null +++ b/internal/link/bus_test.go @@ -0,0 +1,95 @@ +package link + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// **Both implementations, one fixture.** The point of the seam is not that the transport can be +// swapped — it is that the two can be held to the same envelope while both ship, so the day the +// bus moves is a configuration change rather than a discovery. +// +// Against a real server, because what the fixture pins is what reaches the wire: +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestTheNatsBus +func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + f := loadFixture(t, "events/module-event.json") + + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.AddStream(&nats.StreamConfig{ + Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}, + }); err != nil && err != nats.ErrStreamNameAlreadyInUse { + t.Fatal(err) + } + + bus := OverNATS{JS: js} + body, _ := json.Marshal(f.Given.Body) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil { + t.Fatal(err) + } + + // Read back from the stream, not from the thing that wrote it. + raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject) + if err != nil { + t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err) + } + for _, h := range f.Wire.RequiredHeaders { + if raw.Header.Get(h) == "" { + t.Errorf("%s is not set on the wire, and the fixture requires it", h) + } + } + if got := raw.Header.Get("x-source"); got != f.Given.Module { + t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module) + } + if string(raw.Data) != string(body) { + t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body) + } + // The envelope must not also be nested inside the payload. + var nested map[string]any + if json.Unmarshal(raw.Data, &nested) == nil { + if _, has := nested["key"]; has { + t.Error("the payload carries the envelope's own fields, which the fixture refuses") + } + } +} + +// The subject a declaration lands on is one node's, and nothing else's — the state shape. +func TestADeclarationIsAddressedToOneNode(t *testing.T) { + if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" { + t.Fatalf("a declaration would go to %q", got) + } + if DeclareSubject("anchor") == DeclareSubject("laptop") { + t.Fatal("two nodes share a declaration subject, so each would apply the other's") + } +} + +// A module cannot emit under another's name: the subject is derived from the source, and the +// server's permissions make that subject the authority. +func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { + if got := EventSubject("shop", "order.placed"); got != "mesh.mod.shop.event.order.placed" { + t.Fatalf("an event would land on %q", got) + } + if EventSubject("shop", "x") == EventSubject("billing", "x") { + t.Fatal("two modules share an event subject, so neither owns its own name") + } +} diff --git a/internal/link/declare.go b/internal/link/declare.go index 9f6fd7e..665c810 100644 --- a/internal/link/declare.go +++ b/internal/link/declare.go @@ -5,8 +5,6 @@ import ( "encoding/json" "fmt" "time" - - amqp "github.com/rabbitmq/amqp091-go" ) // Signer is whatever holds the control plane's signing key. @@ -21,7 +19,7 @@ type Signer interface { // something else, and the node would refuse a declaration that was genuinely the mesh's. // // Published to the node's own queue, which its account alone may read. -func Declare(ctx context.Context, channel *amqp.Channel, signer Signer, node string, +func Declare(ctx context.Context, bus Bus, signer Signer, node string, declaration []byte, timeout time.Duration) error { if !json.Valid(declaration) { @@ -41,13 +39,5 @@ func Declare(ctx context.Context, channel *amqp.Channel, signer Signer, node str publish, cancel := context.WithTimeout(ctx, timeout) defer cancel() - // Published to the queue directly rather than through the exchange: a declaration is for one - // node, and routing it by name through a shared exchange would mean a binding per node that - // nothing removes when a node is retired. - return channel.PublishWithContext(publish, "", QueueFor(node), false, false, - amqp.Publishing{ - ContentType: "application/json", - DeliveryMode: amqp.Persistent, - Body: body, - }) + return bus.PublishDeclaration(publish, node, body) } diff --git a/internal/link/events.go b/internal/link/events.go index 3d461a3..225a19a 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -6,9 +6,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "time" - - amqp "github.com/rabbitmq/amqp091-go" ) // Emitting a module event from Go. @@ -29,32 +26,17 @@ const ( // EmitEvent publishes one module event, in the envelope the sdk's consumers expect. // -// Persistent, because an event that a broker restart loses is not an announcement. The publish is -// not confirmed here: the caller has already done the work the event describes, and a build that -// succeeded must not be reported as failed because saying so failed. -func EmitEvent(ctx context.Context, channel *amqp.Channel, eventType, source, node string, body any) error { +// The envelope is the transport's to write (bus.go) and this is only what goes in it, which is +// what lets one conformance fixture hold both implementations to the same headers. +func EmitEvent(ctx context.Context, bus Bus, eventType, source, node string, body any) error { payload, err := json.Marshal(body) if err != nil { return fmt.Errorf("cannot serialise a %s event: %w", eventType, err) } - id, err := eventID() - if err != nil { - return err - } - return channel.PublishWithContext(ctx, EventsExchange, eventType, false, false, amqp.Publishing{ - ContentType: "application/json", - DeliveryMode: amqp.Persistent, - MessageId: id, - Timestamp: time.Now().UTC(), - Body: payload, - Headers: amqp.Table{ - "x-event-id": id, - "x-source": source, - "x-node": node, - "x-time": time.Now().UTC().Format(time.RFC3339), - "content-type": "application/json", - }, - }) + // The publish is not confirmed by the caller: it has already done the work the event + // describes, and a build that succeeded must not be reported as failed because saying so + // failed. Each transport decides what "published" means for it. + return bus.PublishEvent(ctx, eventType, source, node, payload) } // eventID is what a consumer deduplicates on: delivery is at-least-once, so a handler must be able diff --git a/internal/link/serve.go b/internal/link/serve.go index 925c237..26ef006 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -631,7 +631,7 @@ func (s *Server) catchingUp(ctx context.Context, delivery amqp.Delivery) { sent := 0 for _, a := range announcements { a.Replay = true - if err := EmitEvent(ctx, s.channel, KeyModuleBuilt, "control-plane", "", a); err != nil { + if err := EmitEvent(ctx, OverAMQP{Channel: s.channel}, KeyModuleBuilt, "control-plane", "", a); err != nil { // Said and abandoned rather than retried: the catalogue asks again every time it // starts, and half a graph delivered twice is no better than half delivered once. s.log.Printf("replaying %s at %s failed, and the rest is abandoned: %v", -- 2.54.0 From 92d87b0082d86d18bcaafe39ccdc01442464719c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:47:26 +0200 Subject: [PATCH 13/39] =?UTF-8?q?gofmt=20bus.go=20=E2=80=94=20import=20gro?= =?UTF-8?q?uping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/link/bus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/link/bus.go b/internal/link/bus.go index a353c58..b222955 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -5,8 +5,8 @@ import ( "fmt" "time" - amqp "github.com/rabbitmq/amqp091-go" "github.com/nats-io/nats.go" + amqp "github.com/rabbitmq/amqp091-go" ) // Bus is what the controller needs of the mesh's bus, **in the mesh's own words rather than a -- 2.54.0 From 6c12780abe7a85f9533e4be75de1507d31dd679c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:51:00 +0200 Subject: [PATCH 14/39] Describe the bus on its own terms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comments framed the new bus by what it replaces — a comparison in almost every explanation, which reads as though NATS were a variant of the old thing rather than the mesh's nervous system. Removed throughout, and OverAMQP becomes OverCurrent: the seam's two sides are the bus the mesh runs on today and the one being built, not two protocols. What remains is the client library's own package name, which is its name. --- cmd/mesh-builder/main.go | 2 +- cmd/mesh-controller/push.go | 8 ++++---- internal/broker/nats.go | 22 +++++++++------------- internal/link/bus.go | 25 ++++++++++++------------- internal/link/serve.go | 2 +- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index c6266f7..f321436 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -268,7 +268,7 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis "manifest": json.RawMessage(result.Manifest), "against": result.Against, "made": result.Made, } - if err := link.EmitEvent(publishCtx, link.OverAMQP{Channel: channel}, link.KeyModuleBuilt, "builder", on, announced); err != nil { + if err := link.EmitEvent(publishCtx, link.OverCurrent{Channel: channel}, link.KeyModuleBuilt, "builder", on, announced); err != nil { // Said, not fatal: the build happened and was answered. A module the catalogue has not // heard of is a gap somebody can close; a build reported as failed because announcing // it failed is a lie about work that was done. diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 7e9c154..2ee6875 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -142,7 +142,7 @@ func declare(ctx context.Context, args []string) error { } defer server.Close() - if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, node, raw, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, node, raw, 15*time.Second); err != nil { return err } fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) @@ -310,7 +310,7 @@ func pushCommand(ctx context.Context, args []string) error { if err != nil { return err } - if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } // After it is away, not before. A digest recorded for something that failed to send would @@ -393,7 +393,7 @@ func pushCommand(ctx context.Context, args []string) error { return declarationWith(held, open, node, plan, settings, gens, Allocating) }, func(s readyNode, body []byte) error { - if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, s.node, body, + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } @@ -611,7 +611,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error { if err != nil { return err } - if err := link.Declare(ctx, link.OverAMQP{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } record, err := inv.NodeByName(ctx, s.node) diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 71c3356..4a09ba3 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -1,19 +1,15 @@ // Composing the bus's own configuration. // -// On AMQP an account was made by calling the broker's management API (management.go). On NATS it -// is *composed*: the controller writes accounts, users and per-subject permissions into one file -// the host keeps current, and the server reloads it in place (novox/hq ADR 0106 — never through a -// management API; design 25 §4). +// An account is *composed*, never called for: the controller writes accounts, users and +// per-subject permissions into one file the host keeps current, and the server reloads it in +// place (novox/hq ADR 0106 — never through a management API; design 25 §4). // // Everything here is pure. Given the principals, it returns the file's text — so the whole of the -// mesh's authority model is testable as strings, with no server, which is what management.go's -// `modulePermissions` already did for the half of it that could be. +// mesh's authority model is testable as strings, with no server. // -// **NATS closes a gap AMQP left open.** management.go records it plainly: LavinMQ has no topic -// permissions, so an emitting module is granted the events exchange whole, and ADR 0042's origin -// reservation — a module publishes only under its own name — is "stamped by the sdk, not enforced -// here". NATS permissions are per subject, so that reservation becomes something the server -// refuses rather than something a library promises. +// **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042 +// reserves a module's origin — it publishes only under its own name — and here that is a refusal +// rather than something a library promises. package broker import ( @@ -270,8 +266,8 @@ func consumerDurable(p Principal) string { // Server is everything the composed file needs that is not a principal. type Server struct { - // ClientPort carries TLS itself; there is no plaintext port beside it, which is where this - // differs from the AMQP broker's 5671/5672 pair. + // ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable + // without TLS is one a module can reach without TLS by mistake. ClientPort int MonitoringPort int TLSCert string diff --git a/internal/link/bus.go b/internal/link/bus.go index b222955..8b767e3 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -12,15 +12,14 @@ import ( // Bus is what the controller needs of the mesh's bus, **in the mesh's own words rather than a // transport's** (novox/hq ADR 0116 step 3). // -// Until now every one of these functions took an `*amqp.Channel`, so the transport reached every -// caller and swapping it meant touching all of them. The seam is small — the controller sends -// exactly two kinds of message that expect no answer, and asks two kinds of question — which is -// why the bus could be replaced at all. +// Until now every one of these functions took the transport's own channel type, so the transport +// reached every caller and changing it meant touching all of them. The seam is small — the +// controller sends exactly two kinds of message that expect no answer, and asks two kinds of +// question — which is why the bus can be replaced at all. // -// Two implementations live below. Both ship: steps 1 to 4 leave every node on AMQP -// ([ADR 0116](novox/hq)), so the controller keeps speaking it and the NATS one is selected at the -// rollout. That is also what makes them comparable — the same caller, the same arguments, and a -// conformance fixture holding both to one envelope. +// Two implementations live below, and both ship until the rollout (ADR 0116: nothing moves a +// node's bus before step 5). Both shipping is what makes them comparable — the same caller, the +// same arguments, and one conformance fixture holding them to one envelope. type Bus interface { // PublishEvent announces something that happened, under the emitter's own name. 1:many, and // nobody is obliged to act (ADR 0041). @@ -32,12 +31,12 @@ type Bus interface { PublishDeclaration(ctx context.Context, node string, body []byte) error } -// --- AMQP, the bus the mesh runs on today ----------------------------------------------------- +// --- The bus the mesh runs on today ----------------------------------------------------- -// OverAMQP is the bus as a channel. -type OverAMQP struct{ Channel *amqp.Channel } +// OverCurrent is the bus the mesh runs on today, until the rollout. +type OverCurrent struct{ Channel *amqp.Channel } -func (b OverAMQP) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { +func (b OverCurrent) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { id, err := eventID() if err != nil { return err @@ -58,7 +57,7 @@ func (b OverAMQP) PublishEvent(ctx context.Context, key, source, node string, bo }) } -func (b OverAMQP) PublishDeclaration(ctx context.Context, node string, body []byte) error { +func (b OverCurrent) PublishDeclaration(ctx context.Context, node string, body []byte) error { // To the queue directly rather than through an exchange: a declaration is for one node, and // routing it by name through a shared exchange would mean a binding per node that nothing // removes when a node is retired. diff --git a/internal/link/serve.go b/internal/link/serve.go index 26ef006..b61f633 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -631,7 +631,7 @@ func (s *Server) catchingUp(ctx context.Context, delivery amqp.Delivery) { sent := 0 for _, a := range announcements { a.Replay = true - if err := EmitEvent(ctx, OverAMQP{Channel: s.channel}, KeyModuleBuilt, "control-plane", "", a); err != nil { + if err := EmitEvent(ctx, OverCurrent{Channel: s.channel}, KeyModuleBuilt, "control-plane", "", a); err != nil { // Said and abandoned rather than retried: the catalogue asks again every time it // starts, and half a graph delivered twice is no better than half delivered once. s.log.Printf("replaying %s at %s failed, and the rest is abandoned: %v", -- 2.54.0 From d0a9abcb1cb4451d70112b97729953e77c02f164 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:05:52 +0200 Subject: [PATCH 15/39] The store window as a decision, and the problem moving it to the server introduces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3.4, the consume side's hard part. The guarantee (ADR 0083) is that a push the controller cannot record because its store is restarting is held and retried — never dropped, never falsely acknowledged. Keeping the delivery unacknowledged in memory becomes a nak with a delay: the server holds it, the controller keeps no list of parked messages, and a controller that restarts mid-window loses nothing it was holding. That is a plain win and it introduces one problem. Holding in memory let the controller drop an older report when a newer one for the same node arrived, "because acting on it after the newer would undo the newer". A naked message is the server's and comes back whatever happened meanwhile, so the older report is redelivered after the newer was applied. The answer was already in the message. A report carries `Declared`, the digest of the declaration it is about, which exists because an earlier attempt to order reports by time lost the race it invited. So supersession stops being something the controller remembers and becomes something it checks — the same shape as a node refusing a superseded declaration by sequence (issue 107): ordering settled by what a message says, not by when it arrived. Pure, so the guarantee is testable without a bus, a store or a clock. Nine tests, including that staleness is decided before the store is waited on — a redelivery that lost its race must not hold a slot a current message needs. --- internal/link/window.go | 106 +++++++++++++++++++++++++++++++++++ internal/link/window_test.go | 84 +++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 internal/link/window.go create mode 100644 internal/link/window_test.go diff --git a/internal/link/window.go b/internal/link/window.go new file mode 100644 index 0000000..2b89a15 --- /dev/null +++ b/internal/link/window.go @@ -0,0 +1,106 @@ +package link + +import ( + "errors" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// The store window, as a decision rather than a mechanism. +// +// The guarantee (novox/hq ADR 0083): a push the controller cannot record because its store is +// restarting is **held and retried**, not dropped and not falsely acknowledged. On the bus the +// mesh runs on today that is done by keeping the delivery unacknowledged in memory and settling +// it later. On the bus being built it is a `nak` with a delay: the server holds it and redelivers, +// so the controller keeps no list of parked messages and a controller that restarts mid-window +// loses nothing it was holding. +// +// **The decision is the same either way, and the mechanism is not the interesting part.** What is +// interesting is that moving the holding into the server introduces a problem the in-memory +// version did not have, and the answer was already in the message. + +// Verdict is what to do with one control message. +type Verdict int + +const ( + // Take it: apply, then acknowledge. + Take Verdict = iota + // Hold it: the store cannot record this yet. Nak with a delay and let the server redeliver. + Hold + // Stale: this is about a declaration the node has already moved past, and applying it would + // undo what came after. Acknowledge without acting — redelivering forever is worse. + Stale + // GiveUp: the store has not come back in time. Settle it and say so, loudly. + GiveUp +) + +// StoreWindow decides. Pure, so the guarantee is testable without a bus, a store or a clock. +type StoreWindow struct { + // GiveUpAfter is how long one message may be held before it is let go with a line saying so. + GiveUpAfter time.Duration +} + +// Decide answers for one delivery. +// +// - err is what the store said, or nil. +// - declaredIn is the digest of the declaration this message is about, empty when it is not +// about one (an enrolment, a build result). +// - outstanding is the digest the mesh last sent that node, empty when it has sent none. +// - heldFor is how long this message has already been held; zero on first delivery. +func (w StoreWindow) Decide(err error, declaredIn, outstanding string, heldFor time.Duration) Verdict { + // **Staleness is checked before the store, not after.** A redelivery that lost its race is + // not worth waiting on a store for, and asking the store first would mean a message about a + // superseded declaration holding a slot in the window that a current one needs. + if declaredIn != "" && outstanding != "" && declaredIn != outstanding { + return Stale + } + if err == nil { + return Take + } + if !errors.Is(err, ErrTryAgain) && !inventory.Unreachable(err) { + // Not the store being away: a refusal is an answer, and holding it would turn a message + // the mesh understood into one it retries forever. + return Take + } + if heldFor >= w.GiveUpAfter { + return GiveUp + } + return Hold +} + +// RedeliverAfter is how long the server should hold a naked message before trying again. +// +// Backed off, and bounded. A store restarting is back in seconds; a store that is gone is not +// helped by being asked every second, and the delay is what keeps a window of held messages from +// becoming a spin. +func RedeliverAfter(heldFor time.Duration) time.Duration { + switch { + case heldFor < 5*time.Second: + return time.Second + case heldFor < 30*time.Second: + return 5 * time.Second + default: + return 15 * time.Second + } +} + +// The problem holding-in-the-server introduces, and why the answer was already in the message. +// +// Holding a delivery in memory let the controller do something a server cannot: when a newer +// report for the same node arrived, it dropped the older one, "because acting on it after the +// newer would undo the newer". A `nak`ed message is the server's, and the server will redeliver +// it whatever else has happened in the meantime — so the older report comes back *after* the +// newer was applied, and applying it would undo exactly what that comment describes. +// +// **A report already says which declaration it is about.** `Declared` is the digest of the exact +// bytes the mesh sent, and it exists because an earlier attempt to order reports by time lost the +// race it invited — an apply that started under the previous declaration finishes after the next +// is sent, and the report reads as newer than the send. Clocks cannot answer *which*; the digest +// is the answer itself. +// +// So supersession stops being a thing the controller remembers and becomes a thing it checks: a +// report whose digest is not the one outstanding for that node is stale, and is acknowledged +// without being acted on. Which is the same shape as a node refusing a superseded declaration by +// sequence (novox/hq issue 107) — ordering settled by what the message says, not by when it +// happened to arrive. diff --git a/internal/link/window_test.go b/internal/link/window_test.go new file mode 100644 index 0000000..531efea --- /dev/null +++ b/internal/link/window_test.go @@ -0,0 +1,84 @@ +package link + +import ( + "errors" + "testing" + "time" +) + +func window() StoreWindow { return StoreWindow{GiveUpAfter: 2 * time.Minute} } + +// The guarantee itself (novox/hq ADR 0083): a push the store cannot record is held, not dropped +// and not falsely acknowledged. +func TestAMessageTheStoreCannotTakeYetIsHeld(t *testing.T) { + if got := window().Decide(ErrTryAgain, "", "", 0); got != Hold { + t.Fatalf("got %v; a push the store could not record was not held", got) + } +} + +// A refusal is an answer. Holding it would turn a message the mesh understood into one it +// retries forever. +func TestARefusalIsNotHeld(t *testing.T) { + if got := window().Decide(errors.New("that node does not exist"), "", "", 0); got != Take { + t.Fatalf("got %v; a refusal was mistaken for the store being away", got) + } +} + +// Held has a limit, and past it the message is settled rather than held for ever. +func TestAStoreThatNeverComesBackEndsTheHold(t *testing.T) { + if got := window().Decide(ErrTryAgain, "", "", 3*time.Minute); got != GiveUp { + t.Fatalf("got %v; the window has no end", got) + } + if got := window().Decide(ErrTryAgain, "", "", time.Minute); got != Hold { + t.Fatalf("got %v; the window ended early", got) + } +} + +// **The problem that holding in the server introduces.** A naked message is redelivered whatever +// else happened meanwhile, so a report about a superseded declaration comes back after the newer +// one was applied — and applying it would undo the newer. +func TestAReportAboutASupersededDeclarationIsNotApplied(t *testing.T) { + got := window().Decide(nil, "digest-of-the-old-one", "digest-of-the-current-one", 0) + if got != Stale { + t.Fatalf("got %v; a redelivery that lost its race would have undone what came after", got) + } +} + +func TestAReportAboutTheOutstandingDeclarationIsApplied(t *testing.T) { + if got := window().Decide(nil, "same", "same", 0); got != Take { + t.Fatalf("got %v; a current report was discarded", got) + } +} + +// A message that is about no declaration — an enrolment, a build result — is never stale: there +// is nothing for it to be out of date with. +func TestAMessageAboutNoDeclarationIsNeverStale(t *testing.T) { + if got := window().Decide(nil, "", "whatever-is-outstanding", 0); got != Take { + t.Fatalf("got %v; an enrolment was treated as a stale report", got) + } + if got := window().Decide(nil, "a-digest", "", 0); got != Take { + t.Fatalf("got %v; a report was called stale against a node that was sent nothing", got) + } +} + +// Staleness is decided before the store is waited on: a redelivery that lost its race must not +// hold a slot in the window that a current message needs. +func TestAStaleMessageIsNotHeldForTheStore(t *testing.T) { + if got := window().Decide(ErrTryAgain, "old", "current", 0); got != Stale { + t.Fatalf("got %v; a superseded message was held for a store it would never be applied to", got) + } +} + +// The delay backs off: a store that is gone is not helped by being asked every second, and the +// delay is what keeps a window of held messages from becoming a spin. +func TestRedeliveryBacksOff(t *testing.T) { + first := RedeliverAfter(0) + later := RedeliverAfter(10 * time.Second) + last := RedeliverAfter(time.Minute) + if !(first < later && later < last) { + t.Fatalf("delays do not back off: %v %v %v", first, later, last) + } + if last > 30*time.Second { + t.Fatalf("a held message waits %v between attempts, which is longer than a store restart", last) + } +} -- 2.54.0 From cc019908c2b5c002a214cf725492c353f0fa9560 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:11:30 +0200 Subject: [PATCH 16/39] Asking a tool goes through the seam, and loses two problems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3.4. On the new bus there is no reply queue to declare and no correlation to check: each account is granted one inbox prefix and no other, so an answer cannot reach the wrong asker. That settles a cost build.go records having paid — on a shared reply exchange every asker saw every result, which is why the correlation was checked rather than assumed. And a tool nobody serves says so at once rather than after the whole wait. The difference between "that module is down" and "that tool is slow" is the first thing a person asking wants, and both tests are against a real server because both are claims about what the server does, not about this code. RequestBuild stays as it is, and is a different shape on the new bus rather than the same one: a build takes minutes, so it is work submitted to a queue with the outcome returning to a reply subject the request carries — the pattern design 25 §2 already sets for anything crossing a stream. It touches the builder too, so it goes with that conversion. --- internal/link/bus.go | 48 ++++++++++++++++++++++++++++++- internal/link/bus_test.go | 59 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/internal/link/bus.go b/internal/link/bus.go index 8b767e3..d35b864 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -2,6 +2,7 @@ package link import ( "context" + "errors" "fmt" "time" @@ -29,6 +30,11 @@ type Bus interface { // declaration is not an event, and replaying yesterday's is actively harmful // (design 29 §4, the *state* shape). PublishDeclaration(ctx context.Context, node string, body []byte) error + + // AskTool sends one question to a module's tool and awaits one answer. A tool nobody serves + // must say so **at once** rather than after the whole wait: the difference between "that + // module is down" and "that tool is slow" is the first thing a person asking wants. + AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) } // --- The bus the mesh runs on today ----------------------------------------------------- @@ -57,6 +63,16 @@ func (b OverCurrent) PublishEvent(ctx context.Context, key, source, node string, }) } +// AskTool is implemented over the existing reply-queue machinery in ask.go; this seam does not +// change how it works today. +func (b OverCurrent) AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) { + answer, err := Ask(ctx, b.Channel, module, tool, args, timeout) + if err != nil { + return nil, err + } + return answer.Result, nil +} + func (b OverCurrent) PublishDeclaration(ctx context.Context, node string, body []byte) error { // To the queue directly rather than through an exchange: a declaration is for one node, and // routing it by name through a shared exchange would mean a binding per node that nothing @@ -71,7 +87,10 @@ func (b OverCurrent) PublishDeclaration(ctx context.Context, node string, body [ // --- NATS, the bus being built ---------------------------------------------------------------- // OverNATS is the bus as a JetStream context. -type OverNATS struct{ JS nats.JetStreamContext } +type OverNATS struct { + Conn *nats.Conn + JS nats.JetStreamContext +} // EventSubject is where a module's event lands. Derived from the emitter, never taken from the // caller: a source that could differ from the subject is an envelope that can lie about its @@ -118,3 +137,30 @@ func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []by } return nil } + +// ToolSubject is where a module answers. Derived from the module and the tool, so a caller names +// what it wants rather than where it lives. +func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool } + +func (b OverNATS) AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) { + if len(args) == 0 { + args = []byte(`{}`) + } + ask, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // **No reply queue, and no correlation to check.** The caller's inbox is its own — each + // account is granted one prefix and no other (design 25 §4) — so an answer cannot reach the + // wrong asker and there is nothing to correlate against. That also settles a cost recorded + // in build.go: on a shared reply exchange every asker saw every result. + msg, err := b.Conn.RequestWithContext(ask, ToolSubject(module, tool), args) + if err != nil { + if errors.Is(err, nats.ErrNoResponders) { + // Said at once rather than after the whole wait: nothing is subscribed to that + // subject, which is a different fact from a tool being slow. + return nil, fmt.Errorf("nothing serves %s.%s", module, tool) + } + return nil, fmt.Errorf("asking %s.%s: %w", module, tool, err) + } + return msg.Data, nil +} diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go index afd5f94..909c4db 100644 --- a/internal/link/bus_test.go +++ b/internal/link/bus_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "os" + "strings" "testing" "time" @@ -93,3 +94,61 @@ func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { t.Fatal("two modules share an event subject, so neither owns its own name") } } + +// A tool nobody serves says so at once. The difference between "that module is down" and "that +// tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it +// is how a fast answer becomes a slow non-answer. +func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + bus := OverNATS{Conn: conn} + began := time.Now() + _, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second) + if err == nil { + t.Fatal("asking a tool nothing serves succeeded") + } + if took := time.Since(began); took > 2*time.Second { + t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took) + } + if !strings.Contains(err.Error(), "nothing serves") { + t.Errorf("the refusal does not say nobody is there: %v", err) + } +} + +// And a served tool answers, with no reply queue to declare and no correlation to check. +func TestAskingAServedToolAnswers(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) { + _ = m.Respond([]byte(`{"total":12}`)) + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + + got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price", + []byte(`{"qty":4}`), 5*time.Second) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"total":12}` { + t.Fatalf("the answer came back as %s", got) + } +} -- 2.54.0 From abce68fdf02ea9a4700dcdf3f6575851ef4190e8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:14:32 +0200 Subject: [PATCH 17/39] Verify that a reply address does not survive a stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 25 §2 builds enrolment around carrying the reply subject in the payload, because a JetStream consumer claims the transport Reply field for its own ack address. The whole handshake rests on it, so it is checked: the caller asked for _INBOX.LCr3M83q... and the consumer saw $JS.ACK.PROBE.probe_consumer... The design was right, and the workaround is necessary rather than defensive. Worth having as a test rather than a note: if a future server version stopped doing this, enrolment would keep working and the reason for the payload field would quietly become folklore. --- internal/link/enrol_reply_probe_test.go | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 internal/link/enrol_reply_probe_test.go diff --git a/internal/link/enrol_reply_probe_test.go b/internal/link/enrol_reply_probe_test.go new file mode 100644 index 0000000..7f17d09 --- /dev/null +++ b/internal/link/enrol_reply_probe_test.go @@ -0,0 +1,81 @@ +package link + +import ( + "os" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// **Verified 2026-09-27**: the caller asked for a reply to `_INBOX.LCr3M83q…` and the consumer +// saw `$JS.ACK.PROBE.probe_consumer.1.1.1…`. The design was right, and enrolment's payload-borne +// reply subject is necessary rather than defensive. +// +// Design 25 §2 asserts that a reply address is **eaten** when a message crosses a stream: core +// request/reply puts the requester's inbox in the message's Reply field, but a JetStream consumer +// has already claimed that field for its own ack address by the time a handler sees it. The whole +// enrolment design rests on it — the reply subject travels in the payload instead — so it is +// checked rather than believed. +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestAReplyAddress +func TestAReplyAddressDoesNotSurviveAStream(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.AddStream(&nats.StreamConfig{ + Name: "PROBE", Subjects: []string{"probe.>"}, Retention: nats.WorkQueuePolicy, + }); err != nil && err != nats.ErrStreamNameAlreadyInUse { + t.Fatal(err) + } + defer js.DeleteStream("PROBE") + + seen := make(chan *nats.Msg, 1) + sub, err := js.Subscribe("probe.enrol", func(m *nats.Msg) { seen <- m }, + nats.Durable("probe_consumer"), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + + // A caller doing what core request/reply does: publish with its own inbox as the reply. + inbox := nats.NewInbox() + if err := conn.PublishMsg(&nats.Msg{Subject: "probe.enrol", Reply: inbox, Data: []byte("{}")}); err != nil { + t.Fatal(err) + } + + select { + case m := <-seen: + t.Logf("the caller asked for a reply to %s", inbox) + t.Logf("the consumer sees a Reply field of %s", m.Reply) + if m.Reply == inbox { + t.Fatalf("the reply address SURVIVED the stream. Design 25 §2 says it does not, and "+ + "builds enrolment around carrying the reply subject in the payload to work "+ + "around it. If this holds generally, that work is unnecessary and the design "+ + "should say so.") + } + if m.Reply == "" { + t.Fatal("the Reply field is empty rather than claimed; the design says it carries " + + "the consumer's ack address, which is a different fact") + } + // Answering it would send the enrolling node's credentials to an ack subject. + if len(m.Reply) < 7 || m.Reply[:7] != "$JS.ACK" { + t.Errorf("the Reply field is %q, which is neither the caller's inbox nor an ack "+ + "address — the design's reasoning assumes one of the two", m.Reply) + } + m.Ack() + case <-time.After(5 * time.Second): + t.Fatal("nothing was delivered") + } +} -- 2.54.0 From ce6ac057f609298fa6c437e1e0a92eac1e604ac8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:14:47 +0200 Subject: [PATCH 18/39] gofmt the probe --- internal/link/enrol_reply_probe_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/link/enrol_reply_probe_test.go b/internal/link/enrol_reply_probe_test.go index 7f17d09..f00a879 100644 --- a/internal/link/enrol_reply_probe_test.go +++ b/internal/link/enrol_reply_probe_test.go @@ -60,9 +60,9 @@ func TestAReplyAddressDoesNotSurviveAStream(t *testing.T) { t.Logf("the caller asked for a reply to %s", inbox) t.Logf("the consumer sees a Reply field of %s", m.Reply) if m.Reply == inbox { - t.Fatalf("the reply address SURVIVED the stream. Design 25 §2 says it does not, and "+ - "builds enrolment around carrying the reply subject in the payload to work "+ - "around it. If this holds generally, that work is unnecessary and the design "+ + t.Fatalf("the reply address SURVIVED the stream. Design 25 §2 says it does not, and " + + "builds enrolment around carrying the reply subject in the payload to work " + + "around it. If this holds generally, that work is unnecessary and the design " + "should say so.") } if m.Reply == "" { -- 2.54.0 From 7a8a19b11bf90bfcd271d0207118fbedbf83ddad Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:17:52 +0200 Subject: [PATCH 19/39] A person's account (step 4.4, the account half) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 25 §7. A person is not a module and holds no seat: nothing is addressed to them, nothing is delivered to them, and they have no durable consumer. What they have is permission to ask, as a list of tools or `*` for an administrator. Four properties the tests hold it to, each of which is a way of being wrong that would not announce itself: a person reaches nothing but tools, so one cannot claim a module said something; no ack subject, because authority over a consumer that does not exist is authority nobody would audit; no allow_responses, because a person who can answer a request is impersonating a module on a bus where anyone may serve a tool; and two people do not share an inbox. --- internal/broker/nats.go | 36 +++++++++++++++++ internal/broker/nats_test.go | 77 ++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 4a09ba3..80089fd 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -29,6 +29,10 @@ const ( KindNode Kind = "node" KindController Kind = "controller" KindEnrolment Kind = "enrolment" + // KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its + // authority is a list of tools and nothing else — not control, not declarations, not builds, + // and no ability to answer anything, because a person asks. + KindPerson Kind = "person" ) // Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it @@ -56,6 +60,14 @@ type Principal struct { Holds []Seat Uses []Seat + // Invokes are the tools a person may call, as `.`; a single `*` is every tool, + // for an administrator. Only meaningful for KindPerson. + // + // **A list, not a role.** A person is not a module and holds no seat: nothing is addressed + // to them, nothing is delivered to them, and they have no durable consumer to acknowledge. + // What they have is permission to ask. + Invokes []string + // PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal // and never appears here: this file is written to a node's disk and read by a server, and a // secret that can be read from a configuration file is a secret with a wider blast radius @@ -73,6 +85,8 @@ var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) // applies, kept. func (p Principal) Username() string { switch p.Kind { + case KindPerson: + return "person." + p.Module case KindModule: return p.Node + "." + p.Module case KindNode: @@ -129,6 +143,22 @@ func PermissionsFor(p Principal) (Permissions, error) { pub = []string{"mesh.control.>", "mesh.node.>", "mesh.build.>", "$JS.API.>"} sub = []string{"mesh.control.>", "mesh.build.>", "$JS.API.>"} + case KindPerson: + // Tools, and nothing else. Every subject a person may publish is a tool call; a person + // who could publish an event would be able to claim a module said something. + for _, t := range p.Invokes { + if t == "*" { + pub = append(pub, "mesh.mod.*.tool.>") + continue + } + module, tool, ok := strings.Cut(t, ".") + if !ok { + return Permissions{}, fmt.Errorf( + "%q does not name a tool: a person invokes ., or * for every one", t) + } + pub = append(pub, "mesh.mod."+module+".tool."+tool) + } + case KindEnrolment: // A leaked token is useless for anything but enrolling: it cannot read a declaration, hear // an event, or subscribe any inbox but the one its own token derives (design 25 §6). @@ -190,6 +220,12 @@ func PermissionsFor(p Principal) (Permissions, error) { } } + if p.Kind == KindPerson { + // An inbox to hear answers in, and nothing else. No ack subject: a person has no durable + // consumer, because nothing is delivered to a person — they ask and are answered. + sub = append(sub, p.inbox()) + } + if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController { // Its own reply space, and nothing wider. sub = append(sub, p.inbox()) diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index 64193d8..f1aa154 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -164,3 +164,80 @@ func TestAUserWithoutAPasswordIsRefused(t *testing.T) { t.Fatal("composed a user with no password hash") } } + +// A person reaches the mesh's tools from a workstation (design 25 §7). Their authority is a list +// of tools and nothing else. +func TestAPersonMayAskOnlyTheToolsTheyWereGiven(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"shop.price", "telegram.status"}, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + has(t, perms.Publish, "mesh.mod.shop.tool.price") + has(t, perms.Publish, "mesh.mod.telegram.tool.status") + hasNot(t, perms.Publish, "mesh.mod.shop.tool.refund") + hasNot(t, perms.Publish, "mesh.mod.*.tool.>") +} + +// An administrator gets every tool, which is a different grant and looks like one. +func TestAnAdministratorMayAskAnyTool(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + has(t, perms.Publish, "mesh.mod.*.tool.>") +} + +// **Nothing but tools.** A person who could publish an event would be able to claim a module +// said something; one who could publish control traffic would be a second controller. +func TestAPersonReachesNothingButTools(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + for _, p := range perms.Publish { + if !strings.Contains(p, ".tool.") { + t.Errorf("a person may publish %q, which is not a tool call", p) + } + } + for _, s := range perms.Subscribe { + if !strings.HasPrefix(s, "_INBOX.person.") { + t.Errorf("a person may subscribe %q; only their own inbox should be reachable", s) + } + } +} + +// A person has no durable consumer, because nothing is delivered to a person — so no ack +// subject, and an ack permission would be authority over something that does not exist. +func TestAPersonHasNoAckSubject(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + for _, p := range perms.Publish { + if strings.HasPrefix(p, "$JS.ACK") { + t.Errorf("a person was granted %q, and has no consumer to acknowledge", p) + } + } +} + +// A person asks and is answered; they never answer. allow_responses would let a person reply to +// a request — which, on a bus where anyone may serve a tool, is somebody impersonating a module. +func TestAPersonMayNotAnswer(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + if perms.AllowResponses { + t.Fatal("a person may answer a request, which is impersonating a module") + } +} + +// Two people do not share an inbox, or one would read the other's answers. +func TestTwoPeopleDoNotShareAnInbox(t *testing.T) { + a, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"}) + b, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "sam", Invokes: []string{"*"}, PasswordHash: "x"}) + if a.Subscribe[0] == b.Subscribe[0] { + t.Fatalf("both read %s", a.Subscribe[0]) + } +} + +// A malformed grant is refused rather than widened into something that happens to parse. +func TestAToolGrantThatNamesNoToolIsRefused(t *testing.T) { + if _, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"shop"}, PasswordHash: "x"}); err == nil { + t.Fatal("a grant naming a module but no tool was accepted") + } +} -- 2.54.0 From 06cf3c04e58c0bec05dbb975da206322f2b35583 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:44:16 +0200 Subject: [PATCH 20/39] The consume side behind a seam, and the window wiring into the loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The outbound half went behind `Bus` and the transport stopped reaching its callers; this is the other half, and the larger one. Every handler took `amqp.Delivery`, so the serving loop could not move to another bus without moving enrolment, reports, builds, upgrades and catch-up with it in one breath. `Control` states one message in the mesh's words — took it, dropped it, or held it for the store — and `Inbound` is where messages come from. The AMQP implementation is today's loop moved rather than changed: same queues, same prefetch, same holding, because the mesh is running on it and a bus nothing speaks yet is no reason to alter the one every node is on. The window (window.go) is now what decides, instead of the conditions that were inlined in the loop. Two things that surfaced in the wiring: **Supersession is asked before the store, not after.** A report about a declaration the mesh has moved past would otherwise wait out a restarting store to be written and then overwrite what the node is doing now. **Half of a report is not about a declaration, and that half is never stale.** What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted node found, a node moving its overlay key — reaches the mesh on a report and nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is coming, because the node said it once. So staleness is asked only of a report that is purely an apply's account. The one thing holding-in-memory can do that holding-in-the-server cannot is named rather than hidden: `About` sets aside a held message when a newer one about the same thing arrives, and the bus being built ignores it because the digest answers the same question. --- internal/inventory/nodes.go | 19 + internal/link/enrolment.go | 11 + internal/link/receive.go | 125 +++++ internal/link/receive_current.go | 317 +++++++++++ internal/link/receive_current_test.go | 71 +++ internal/link/report_retry_test.go | 83 ++- internal/link/serve.go | 744 +++++++++++--------------- internal/link/stale_report_test.go | 135 +++++ internal/link/store_window_test.go | 123 ++--- internal/link/window.go | 16 +- 10 files changed, 1094 insertions(+), 550 deletions(-) create mode 100644 internal/link/receive.go create mode 100644 internal/link/receive_current.go create mode 100644 internal/link/receive_current_test.go create mode 100644 internal/link/stale_report_test.go diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index fb736b1..00eedf6 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -782,6 +782,25 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error { return err } +// Outstanding is the digest of the declaration a machine was last sent, by its name, and empty +// for one that has never been sent anything. +// +// **By name rather than by id**, because the caller is the serving loop and what a node puts in a +// report is its name. Asked of one machine rather than read from Waiting's sweep, because it is +// asked per message: a report names the declaration it is about, and a report about one the mesh +// has already moved past is not acted on (design 25 §3). +func (i *Inventory) Outstanding(ctx context.Context, name string) (string, error) { + var sent string + err := i.store.Pool().QueryRow(ctx, + `select coalesce(sent, '') from node where name = $1`, name).Scan(&sent) + if errors.Is(err, pgx.ErrNoRows) { + // Not an error worth carrying up: a report from a machine the mesh has no record of has + // nothing to be stale against, and whatever is wrong with it is the listener's to say. + return "", nil + } + return sent, err +} + // Waiting is every machine whose declaration has changed since it was last sent one. // // The caller works out what each machine should be now, because only it can — resolution is the diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 54b8967..657ec21 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -234,6 +234,17 @@ var ErrNoBrokerManagement = errors.New("no broker management configured") // A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is // its own account of its own machine, kept as a copy for recovery — so this writes it down and // decides nothing from it. +// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not +// acted on (design 25 §3, window.go). +// +// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great +// many things that record reports have no idea what was sent — every test in this package among +// them. So the serving loop asks for this when the listener happens to be able to answer, and +// where it cannot, a report has nothing to be stale against and is simply acted on. +func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) { + return e.Inventory.Outstanding(ctx, node) +} + func (e Enrolment) Heard(ctx context.Context, report Report) (err error) { // A store that could not be asked right now is said as such, so the report is kept for // another attempt rather than acknowledged and lost (novox/hq issue 082). diff --git a/internal/link/receive.go b/internal/link/receive.go new file mode 100644 index 0000000..eda491c --- /dev/null +++ b/internal/link/receive.go @@ -0,0 +1,125 @@ +package link + +import ( + "context" + "time" +) + +// The consume side of the bus, in the mesh's own words. +// +// The outbound half went behind `Bus` (bus.go) and the transport stopped reaching its callers. +// This is the other half, and it is the larger one: everything a node or a module says arrives +// here, and until now every handler took the transport's own delivery type — so the serving loop +// could not be moved to another bus without moving enrolment, reports, builds, upgrades and +// catch-up with it in one breath. +// +// **Two implementations, both shipping** (novox/hq ADR 0116: nothing moves a node's bus before +// step 5). Both shipping is what makes them comparable, and it is what lets the store-window +// guarantee (ADR 0083) be stated once — in window.go, pure — rather than twice, once per +// transport, where the two would eventually disagree about the thing that matters most. + +// The kinds of message the controller acts on. +// +// A transport maps its own addressing onto these — a routing key on the bus the mesh has, a +// subject on the one being built — and nothing past this point knows which it was. They are never +// on the wire: the wire is the transport's business, and a kind that travelled would be a third +// name for the same thing. +const ( + KindEnrolment = "enrolment" + KindReport = "report" + KindHeartbeat = "heartbeat" + KindBuilt = "built" + KindModuleMoved = "module-moved" + KindCatchUp = "catch-up" +) + +// Control is one thing a node or a module said, as the controller must act on it. +// +// **Settling is stated as what the mesh means, not as the transport's verbs.** The two buses +// spell them differently — an ack and a reject against a delivery tag, an ack and a term against +// a stream sequence — and the guarantee is the same either way: `Took` is done with, `Drop` is +// understood and not worth another attempt, and `Hold` is the store window, where the message is +// kept and comes back. +// +// A handler that returns without calling any of the three leaves the message unsettled on +// purpose. That is the right answer while shutting down: a cancelled context is not an answer +// about a message, and the bus should hand it to whatever consumes next (novox/hq issue 083). +type Control interface { + // Kind is which of the constants above this is. + Kind() string + + // Body is the message itself — the payload alone, never the envelope. + Body() []byte + + // Redelivered says the bus has handed this message over before. An enrolment cares and + // nothing else does: one already spent is not finished a second time. + Redelivered() bool + + // HeldFor is how long this message has been waiting to be taken. Zero on a first delivery. + // + // **Read from the message rather than remembered by the controller.** On the bus being built + // it is the age of the publish, which a controller that restarted mid-window still reads + // correctly — the whole reason the holding moves into the server. On the bus the mesh has it + // is how long this process has held it, which is the most that transport can say. + HeldFor() time.Duration + + // Answer replies to whoever is waiting on this message; only an enrolment expects one. + // + // Each transport knows where its own answer goes, and they do not agree about it: one carries + // a reply queue in the delivery, and on the other the field that would have carried it has + // been claimed by the consumer's own ack subject, so the address travels in the payload + // (design 25 §2, verified). That difference is exactly what this seam exists to keep out of + // the handler. + Answer(ctx context.Context, body []byte) error + + // Took settles the message: acted on, or understood and needing no action. + Took() error + + // About names what this message is about — a node's report, one module's move, one build's + // outcome — and is said before the store is asked. + // + // A transport that holds messages **in memory** uses it to set aside anything older it is + // holding about the same thing: the older is the past, and letting it come back after the + // newer was acted on would undo the newer. + // + // **This is the one thing holding-in-memory can do that holding-in-the-server cannot**, and + // naming it here rather than hiding it is deliberate. On the bus being built the message + // belongs to the server and comes back whatever happened meanwhile, so this is ignored and the + // digest a report carries answers the same question instead (window.go, design 25 §3). + About(what string) + + // Hold keeps the message and asks for it again after the delay — the store window. + Hold(after time.Duration) error + + // Drop settles the message without acting on it: refused, stale, or given up on. It is not + // delivered again. + Drop() error +} + +// Inbound is where control messages come from. +type Inbound interface { + // Also asks for one more kind to be delivered. + // + // **Nothing is subscribed unless something is listening for it.** A durable queue or a + // durable stream consumer that nobody reads fills quietly, and the first symptom is a bus out + // of disk rather than anything about modules. + Also(kind string) error + + // Receive delivers every message to act until the context ends, and says why it stopped. + Receive(ctx context.Context, act func(context.Context, Control)) error + + // Close lets go of whatever the implementation holds. + Close() +} + +// Outstanding answers which declaration the mesh last sent a node — the digest, not the +// declaration. +// +// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh +// has already moved past is not worth holding a slot in the window that a current message needs. +// It is a separate interface from Listener rather than a method on it, because a controller that +// only publishes needs neither and something that records reports need not also be able to say +// what was sent. +type Outstanding interface { + Outstanding(ctx context.Context, node string) (string, error) +} diff --git a/internal/link/receive_current.go b/internal/link/receive_current.go new file mode 100644 index 0000000..efeeae5 --- /dev/null +++ b/internal/link/receive_current.go @@ -0,0 +1,317 @@ +package link + +import ( + "context" + "errors" + "fmt" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The consume side on the bus the mesh runs on today. +// +// Everything here was the serving loop's until the seam went in: the queues, the binds, the +// prefetch, and the list of messages the store could not take yet. It moved rather than changed — +// the behaviour this transport has is the behaviour it had, because the mesh is running on it and +// a bus nothing speaks yet is no reason to alter the one every node is on (ADR 0116). + +// Prefetch is how many messages the bus hands the controller before it has settled them. +// +// More than one because a message the store could not take is held, unsettled, while the loop +// goes on answering others — an enrolment above all, which a host is waiting on (novox/hq issue +// 083). Bounded, because what is held is also what the bus has not kept on its own disk as +// pending. +const Prefetch = 64 + +// PrefetchHeadroom is how much of the prefetch is never held, so the loop always has messages to +// answer — an enrolment above all — while others wait for the store. +const PrefetchHeadroom = 8 + +// TryAgainAfter is how often the held are looked at. A store comes back in seconds, and a report a +// few seconds late is still current. +const TryAgainAfter = 2 * time.Second + +// currentInbound consumes what nodes say over the bus the mesh has. +type currentInbound struct { + conn *amqp.Connection + channel *amqp.Channel + // upgrades and catchups are bound only when something is listening (Also). + upgrades bool + catchups bool + // held is every message the store could not take, by the bus's own delivery tag. Kept here + // rather than in the serving loop because holding a delivery unacknowledged is this + // transport's way of keeping it, and the other's is to hand it back to the server. + held map[uint64]*holding + // again is how often the held are looked at; zero means TryAgainAfter. Set by tests. + again time.Duration +} + +// holding is one message kept for the store, and when to try it again. +type holding struct { + message *currentControl + due time.Time + about string +} + +// Current is the consume side of the bus the mesh runs on today. +func Current(conn *amqp.Connection, channel *amqp.Channel) Inbound { + return ¤tInbound{conn: conn, channel: channel, held: map[uint64]*holding{}} +} + +// Also binds the queue one more kind arrives on. +// +// The kinds nodes publish all share one queue and are bound at Connect, because a node may +// publish any of them and binding one while forgetting another is a message the bus accepts, finds +// no queue for, and drops — the publisher sees success and the consumer sees nothing. The two that +// are events get their own queue each, and only when something is listening. +func (c *currentInbound) Also(kind string) error { + switch kind { + case KindModuleMoved: + if err := c.bindEvent(UpgradeQueue, KeyModuleUpgraded); err != nil { + return err + } + c.upgrades = true + case KindCatchUp: + if err := c.bindEvent(CatchUpQueue, KeyCatchingUp); err != nil { + return err + } + c.catchups = true + default: + return fmt.Errorf("nothing binds a queue for %s on this bus", kind) + } + return nil +} + +func (c *currentInbound) bindEvent(queue, key string) error { + if _, err := c.channel.QueueDeclare(queue, true, false, false, false, nil); err != nil { + return fmt.Errorf("cannot declare the %s queue: %w", queue, err) + } + if err := c.channel.QueueBind(queue, key, EventsExchange, false, nil); err != nil { + return fmt.Errorf("cannot bind %s to %s/%s: %w", queue, EventsExchange, key, err) + } + return nil +} + +func (c *currentInbound) Close() {} + +// Receive consumes until the context ends. +// +// One consumer per queue, deliberately: with two on one queue the bus would round-robin between +// them and each would receive half of what it expects — a fault this project has already had, +// between a module's daemon and its capability server. +func (c *currentInbound) Receive(ctx context.Context, act func(context.Context, Control)) error { + // A bounded prefetch rather than one. The loop still takes messages one at a time; what the + // prefetch buys is that a message the store could not take can be held while the loop goes on + // to the next, instead of every enrolment waiting behind it (novox/hq issue 083). Anything + // held goes back to the bus if the controller stops, because nothing held is acknowledged. + if err := c.channel.Qos(Prefetch, 0, false); err != nil { + return err + } + + deliveries, err := c.channel.ConsumeWithContext(ctx, ControlQueue, "control-plane", + false, false, false, false, nil) + if err != nil { + return err + } + + // Its own queue and its own consumer for each event, for the reason above: two consumers on + // one queue split its messages between them, and an upgrade or a catch-up request going to + // whichever half was not listening is a gap that looks like a working mesh. + var upgrades, catchups <-chan amqp.Delivery + if c.upgrades { + upgrades, err = c.channel.ConsumeWithContext(ctx, UpgradeQueue, "control-plane-upgrades", + false, false, false, false, nil) + if err != nil { + return err + } + } + if c.catchups { + catchups, err = c.channel.ConsumeWithContext(ctx, CatchUpQueue, "control-plane-catchup", + false, false, false, false, nil) + if err != nil { + return err + } + } + + closed := c.conn.NotifyClose(make(chan *amqp.Error, 1)) + + again := c.again + if again == 0 { + again = TryAgainAfter + } + ticker := time.NewTicker(again) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if ctx.Err() != nil { + return nil + } + c.retryHeld(ctx, act) + case delivery, ok := <-catchups: + if !ok { + if catchups != nil { + return errors.New("the bus stopped delivering catch-up requests") + } + continue + } + act(ctx, c.wrap(KindCatchUp, delivery)) + case delivery, ok := <-upgrades: + // A nil channel blocks for ever, so this case simply never fires when nothing is + // listening for upgrades. Closed is different, and means the bus stopped. + if !ok { + if upgrades != nil { + return errors.New("the bus stopped delivering upgrades") + } + continue + } + act(ctx, c.wrap(KindModuleMoved, delivery)) + case reason := <-closed: + // Said rather than returned quietly. A controller whose bus connection dropped is a + // mesh where nothing can be told anything, and the reason is the first thing anybody + // will want. + return fmt.Errorf("the bus connection closed: %v", reason) + case delivery, ok := <-deliveries: + if !ok { + return errors.New("the bus stopped delivering") + } + kind, known := kindOfKey[delivery.RoutingKey] + if !known { + // Rejected without requeue: a message nothing understands will not be understood + // on the next attempt either, and requeuing it would spin. + _ = delivery.Reject(false) + continue + } + act(ctx, c.wrap(kind, delivery)) + } + } +} + +// kindOfKey is how this transport's addressing becomes what the mesh calls a message. +var kindOfKey = map[string]string{ + KeyEnrol: KindEnrolment, + KeyReport: KindReport, + KeyAlive: KindHeartbeat, + KeyBuilt: KindBuilt, + KeyModuleUpgraded: KindModuleMoved, + KeyCatchingUp: KindCatchUp, +} + +func (c *currentInbound) wrap(kind string, delivery amqp.Delivery) *currentControl { + return ¤tControl{kind: kind, delivery: delivery, on: c} +} + +// retryHeld hands every message whose delay has passed back to the loop. Each handler holds it +// again, settles it, or lets it go past the bound. +func (c *currentInbound) retryHeld(ctx context.Context, act func(context.Context, Control)) { + now := time.Now() + due := make([]*currentControl, 0, len(c.held)) + for _, h := range c.held { + if !h.due.After(now) { + due = append(due, h.message) + } + } + for _, m := range due { + if ctx.Err() != nil { + return + } + act(ctx, m) + } +} + +// currentControl is one delivery from the bus the mesh has, as the controller reads it. +type currentControl struct { + kind string + delivery amqp.Delivery + on *currentInbound + // about is what this message is about, as the handler named it; empty until it does. + about string + // first is when this message was first held for the store; zero while it has not been. + first time.Time +} + +func (m *currentControl) Kind() string { return m.kind } +func (m *currentControl) Body() []byte { return m.delivery.Body } +func (m *currentControl) Redelivered() bool { return m.delivery.Redelivered } + +func (m *currentControl) HeldFor() time.Duration { + if m.first.IsZero() { + return 0 + } + return time.Since(m.first) +} + +// Answer publishes to the reply queue the request named. +func (m *currentControl) Answer(ctx context.Context, body []byte) error { + if m.delivery.ReplyTo == "" { + return errors.New("that request named no reply queue, so nothing can be told the answer") + } + return m.on.channel.PublishWithContext(ctx, "", m.delivery.ReplyTo, false, false, + amqp.Publishing{ + ContentType: "application/json", + CorrelationId: m.delivery.CorrelationId, + Body: body, + }) +} + +func (m *currentControl) Took() error { + m.forget() + return m.delivery.Ack(false) +} + +// Drop rejects without requeue: on this bus that is what "understood, and not worth another +// attempt" is spelled as, and it is what feeds a dead-letter queue where one is configured. +func (m *currentControl) Drop() error { + m.forget() + return m.delivery.Reject(false) +} + +// About names what this message is about, and lets go of whatever is held about the same thing: +// the held one is the past, and acting on it after this one would undo this one. Acknowledged +// rather than left to come back, because a held message nothing will act on is a place in the +// prefetch nothing gets back. +func (m *currentControl) About(what string) { + m.about = what + if what == "" { + return + } + for tag, h := range m.on.held { + if h.about != what || tag == m.delivery.DeliveryTag { + continue + } + delete(m.on.held, tag) + _ = h.message.delivery.Ack(false) + } +} + +// Hold keeps the message unacknowledged and sets it aside to be handed back after the delay. +// +// Held no further than the prefetch leaves room: past that the bus would hand the loop nothing +// new — enrolments included — until something held was let go. A message that cannot be held says +// so, and the handler settles it its own way. +func (m *currentControl) Hold(after time.Duration) error { + if m.on.held == nil { + m.on.held = map[uint64]*holding{} + } + if _, already := m.on.held[m.delivery.DeliveryTag]; !already { + if len(m.on.held) >= Prefetch-PrefetchHeadroom { + return fmt.Errorf("%d messages are already held for the store, and holding more "+ + "would stop the queue", len(m.on.held)) + } + m.first = time.Now() + } + m.on.held[m.delivery.DeliveryTag] = &holding{ + message: m, due: time.Now().Add(after), about: m.about, + } + return nil +} + +func (m *currentControl) forget() { + if m.on != nil { + delete(m.on.held, m.delivery.DeliveryTag) + } +} diff --git a/internal/link/receive_current_test.go b/internal/link/receive_current_test.go new file mode 100644 index 0000000..462e3f3 --- /dev/null +++ b/internal/link/receive_current_test.go @@ -0,0 +1,71 @@ +package link + +import ( + "context" + "encoding/json" + "io" + "log" + "testing" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The harness for the consume side on the bus the mesh runs on today. +// +// Messages arrive through the seam, so what these tests exercise is the controller's decision +// about a message and this transport's way of keeping one — which is what the seam separated. A +// fake acknowledger stands in for the bus, because what is asserted is how a message was settled +// and that needs no server. + +// settled is how the bus was told to settle one message. +type settled struct{ acked, nacked, requeued, rejected bool } + +func (a *settled) Ack(uint64, bool) error { a.acked = true; return nil } +func (a *settled) Nack(_ uint64, _ bool, requeue bool) error { + a.nacked, a.requeued = true, requeue + return nil +} +func (a *settled) Reject(uint64, bool) error { a.rejected = true; return nil } + +// unsettled is a message the controller has neither taken nor let go: it is held, and the bus will +// hand it to whatever consumes next if the controller stops. +func (a *settled) unsettled() bool { return !a.acked && !a.nacked && !a.rejected } + +var tag uint64 + +func quiet() *log.Logger { return log.New(io.Discard, "", 0) } + +// serving is a controller with nothing but a way of receiving, ready for a listener, a recorder, +// an upgrader or a replayer to be set on it. +func serving() (*Server, *currentInbound) { + in := ¤tInbound{held: map[uint64]*holding{}} + return &Server{inbound: in, bus: OverCurrent{}, log: quiet()}, in +} + +// sends is one message arriving over this transport, as the controller reads it. +func (c *currentInbound) sends(t *testing.T, to *settled, kind string, v any) Control { + t.Helper() + body, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + tag++ + return ¤tControl{kind: kind, on: c, delivery: amqp.Delivery{ + Acknowledger: to, Body: body, DeliveryTag: tag, + }} +} + +// dueNow brings every held message forward, so a test need not wait out the backoff a real store +// restart would be given (RedeliverAfter). +func (c *currentInbound) dueNow() { + for _, h := range c.held { + h.due = time.Now().Add(-time.Second) + } +} + +// retries hands every held message back to the controller, the way the ticker does. +func (c *currentInbound) retries(ctx context.Context, s *Server) { + c.dueNow() + c.retryHeld(ctx, s.act) +} diff --git a/internal/link/report_retry_test.go b/internal/link/report_retry_test.go index 8a175d7..7549437 100644 --- a/internal/link/report_retry_test.go +++ b/internal/link/report_retry_test.go @@ -2,26 +2,11 @@ package link import ( "context" - "encoding/json" "errors" - "io" - "log" "testing" "time" - - amqp "github.com/rabbitmq/amqp091-go" ) -type saidTo struct{ acked, nacked, requeued bool } - -func (a *saidTo) Ack(uint64, bool) error { a.acked = true; return nil } -func (a *saidTo) Nack(_ uint64, _ bool, requeue bool) error { - a.nacked, a.requeued = true, requeue - return nil -} -func (a *saidTo) Reject(uint64, bool) error { return nil } -func (a *saidTo) unsettled() bool { return !a.acked && !a.nacked } - type heardWith struct{ err error } func (h heardWith) Heard(context.Context, Report) error { return h.err } @@ -31,39 +16,31 @@ type switchable struct{ err error } func (h *switchable) Heard(context.Context, Report) error { return h.err } -var tag uint64 - -func aReport(t *testing.T, to *saidTo, node, declared string) amqp.Delivery { - t.Helper() - body, err := json.Marshal(Report{Node: node, Declared: declared, Applied: []string{"store"}}) - if err != nil { - t.Fatal(err) - } - tag++ - return amqp.Delivery{Acknowledger: to, RoutingKey: KeyReport, Body: body, DeliveryTag: tag} +func aReport(node, declared string) Report { + return Report{Node: node, Declared: declared, Applied: []string{"store"}} } -func quiet() *log.Logger { return log.New(io.Discard, "", 0) } - // A report the store could not take right now is held, unsettled, and recorded when the store is // back; one the store answered no to is acknowledged; one recorded is acknowledged (issue 082, 083). func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) { store := &switchable{err: errors.Join(ErrTryAgain, errors.New("starting up"))} - s := &Server{listener: store, log: quiet()} - held := &saidTo{} - s.handleReport(context.Background(), aReport(t, held, "anchor", "d1")) - if !held.unsettled() || len(s.parked) != 1 { - t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(s.parked)) + s, in := serving() + s.listener = store + held := &settled{} + s.act(context.Background(), in.sends(t, held, KindReport, aReport("anchor", "d1"))) + if !held.unsettled() || len(in.held) != 1 { + t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(in.held)) } store.err = nil - s.retryHeld(context.Background()) - if !held.acked || len(s.parked) != 0 { - t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(s.parked)) + in.retries(context.Background(), s) + if !held.acked || len(in.held) != 0 { + t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(in.held)) } - refused := &saidTo{} - s = &Server{listener: heardWith{err: errors.New("a report named no node")}, log: quiet()} - s.handleReport(context.Background(), aReport(t, refused, "anchor", "d1")) + refused := &settled{} + s, in = serving() + s.listener = heardWith{err: errors.New("a report named no node")} + s.act(context.Background(), in.sends(t, refused, KindReport, aReport("anchor", "d1"))) if !refused.acked || refused.nacked { t.Fatalf("a report the store answered no to was not acknowledged: %+v", refused) } @@ -72,33 +49,35 @@ func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) { // A newer report from the same node supersedes one of its reports still held: recorded after the // newer, the older would overwrite what the node is doing now. func TestANewerReportSupersedesAHeldOneFromTheSameNode(t *testing.T) { - s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))}, log: quiet()} - older, newer, other := &saidTo{}, &saidTo{}, &saidTo{} - s.handleReport(context.Background(), aReport(t, older, "anchor", "d1")) - s.handleReport(context.Background(), aReport(t, other, "laptop", "d7")) - s.handleReport(context.Background(), aReport(t, newer, "anchor", "d2")) + s, in := serving() + s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))} + older, newer, other := &settled{}, &settled{}, &settled{} + s.act(context.Background(), in.sends(t, older, KindReport, aReport("anchor", "d1"))) + s.act(context.Background(), in.sends(t, other, KindReport, aReport("laptop", "d7"))) + s.act(context.Background(), in.sends(t, newer, KindReport, aReport("anchor", "d2"))) if !older.acked { t.Fatalf("the older report was not set aside by the newer: %+v", older) } - if !newer.unsettled() || !other.unsettled() || len(s.parked) != 2 { + if !newer.unsettled() || !other.unsettled() || len(in.held) != 2 { t.Fatalf("the newer report and another node's were not both held: newer %+v other %+v, %d held", - newer, other, len(s.parked)) + newer, other, len(in.held)) } } // A store that has not come back within the bound is not restarting: the report is let go, loudly, // rather than held for ever. func TestAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { - s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))}, - log: quiet(), giveUp: time.Millisecond} - held := &saidTo{} - s.handleReport(context.Background(), aReport(t, held, "anchor", "d1")) + s, in := serving() + s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))} + s.giveUp = time.Millisecond + held := &settled{} + s.act(context.Background(), in.sends(t, held, KindReport, aReport("anchor", "d1"))) if !held.unsettled() { t.Fatalf("the first failure was not held: %+v", held) } time.Sleep(5 * time.Millisecond) - s.retryHeld(context.Background()) - if !held.acked || len(s.parked) != 0 { - t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(s.parked)) + in.retries(context.Background(), s) + if !held.acked || len(in.held) != 0 { + t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(in.held)) } } diff --git a/internal/link/serve.go b/internal/link/serve.go index b61f633..f1c5750 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -7,31 +7,30 @@ import ( "encoding/json" "errors" "fmt" - "github.com/novox/mesh-controller/internal/envfile" - "github.com/novox/mesh-controller/internal/inventory" "log" "os" "time" amqp "github.com/rabbitmq/amqp091-go" + + "github.com/novox/mesh-controller/internal/envfile" ) -// AMQPVar is the control plane's own connection to the broker. +// AMQPVar is the controller's own connection to the bus the mesh runs on today. const AMQPVar = "MESH_BROKER_AMQP" -// Enroller is what the control plane does with an enrolment request. +// Enroller is what the controller does with an enrolment request. // -// An interface so the serving loop can be tested against a real broker without a database, and -// so the two concerns — moving messages, and deciding — stay apart. +// An interface so the serving loop can be tested against a real bus without a database, and so the +// two concerns — moving messages, and deciding — stay apart. type Enroller interface { // Enrol spends the token, records the key, and reports the node's name. The error is // returned to the node as a refusal; it must be the same for every reason a token can fail. Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error) } -// Server consumes what nodes say. -// Listener is what the control plane does with a report. Separate from Enroller so the two can -// be given independently, and so a server that only sends declarations needs neither. +// Listener is what the controller does with a report. Separate from Enroller so the two can be +// given independently, and so a server that only sends declarations needs neither. type Listener interface { Heard(ctx context.Context, report Report) error } @@ -46,106 +45,77 @@ type Recorder interface { Built(ctx context.Context, result BuildResult) error } -type Server struct { - conn *amqp.Connection - channel *amqp.Channel - enroller Enroller - listener Listener - recorder Recorder - log *log.Logger - upgrader Upgrader - replayer Replayer - // Messages the store could not take right now, held unacknowledged and tried again on a - // ticker, by subject (novox/hq issues 082, 083). again is the ticker's interval, zero meaning - // TryAgainAfter; giveUp is how long one is kept, zero meaning GiveUpAfter. - again time.Duration - giveUp time.Duration - parked map[string]*held -} - -// held is one message the store could not take, kept to be tried again. -type held struct { - delivery amqp.Delivery - retry func(context.Context, amqp.Delivery) - what string - first time.Time -} - -// ErrTryAgain marks a listener's failure as "not now": what it was given is worth keeping and -// asking again, as when the store is restarting (novox/hq issue 082). -var ErrTryAgain = errors.New("not now, try again") - -// TryAgainAfter is how often messages the store could not take are tried again. A store comes -// back in seconds, and a report a few seconds late is still current. -const TryAgainAfter = 2 * time.Second - -// GiveUpAfter bounds how long one message is kept trying. A store that has not come back in this -// long is not restarting, and the message is let go with a line saying it was lost. -const GiveUpAfter = 2 * time.Minute - -// Prefetch is how many messages the broker hands the control plane before it has settled them. -// More than one because a message the store could not take is held, unsettled, while the loop goes -// on answering others — an enrolment above all, which a host is waiting on (novox/hq issue 083). -// Bounded, because what is held is also what the broker has not kept on its own disk as pending. -const Prefetch = 64 - -// PrefetchHeadroom is how much of the prefetch is never held, so the loop always has messages to -// answer — an enrolment above all — while others wait for the store. -const PrefetchHeadroom = 8 - -// Records tells the server where to keep build results. -// -// Set after Connect rather than passed to it, because a control plane that only publishes — the -// `build` command, which waits for its own answer — needs a connection and no recorder, and -// making it supply one would have it construct something it never uses. -func (s *Server) Records(r Recorder) { s.recorder = r } - -// Upgrader is what the control plane does when the catalogue says a module moved. +// Upgrader is what the controller does when the catalogue says a module moved. // // An interface for the same reason Enroller is one: deciding what an upgrade means for the // machines running it is a different concern from noticing that one was announced, and only the // first needs a database. type Upgrader interface { // Upgraded is told which module moved and between which commits. An error is logged and the - // message is not requeued: an upgrade the control plane could not act on is not one it will - // act on by being handed the same message again, and a poison message on a durable queue - // would stop every upgrade behind it — except the store unreachable for the moment, which is - // asked again for a bounded time (novox/hq issue 083). + // message is not handed back: an upgrade the controller could not act on is not one it will + // act on by being given the same message again, and a poison message on a durable queue would + // stop every upgrade behind it — except the store unreachable for the moment, which is asked + // again for a bounded time (novox/hq issue 083). Upgraded(ctx context.Context, u Upgraded) error } -// Follows says what to do about upgrades, and binds the queue they arrive on. +// Server acts on what nodes and modules say. // -// **Not bound unless something is listening.** A durable queue bound to every upgrade with no -// consumer fills up quietly, and the first symptom is a broker out of disk rather than anything -// about modules. +// **It holds no transport.** What arrives comes through Inbound and what it publishes goes through +// Bus, so this file is the controller's *decisions* about messages and nothing about wires. The +// connection below is the bus the mesh runs on today, kept because the command line publishes over +// the same one until the rollout (ADR 0116). +type Server struct { + inbound Inbound + bus Bus + conn *amqp.Connection + channel *amqp.Channel + + enroller Enroller + listener Listener + recorder Recorder + upgrader Upgrader + replayer Replayer + + log *log.Logger + // giveUp is how long one message is held for the store; zero means GiveUpAfter. + giveUp time.Duration +} + +// ErrTryAgain marks a listener's failure as "not now": what it was given is worth keeping and +// asking again, as when the store is restarting (novox/hq issue 082). +var ErrTryAgain = errors.New("not now, try again") + +// GiveUpAfter bounds how long one message is kept trying. A store that has not come back in this +// long is not restarting, and the message is let go with a line saying it was lost. +const GiveUpAfter = 2 * time.Minute + +// Records tells the server where to keep build results. +// +// Set after Connect rather than passed to it, because a controller that only publishes — the +// `build` command, which waits for its own answer — needs a connection and no recorder, and making +// it supply one would have it construct something it never uses. +func (s *Server) Records(r Recorder) { s.recorder = r } + +// Follows says what to do about upgrades, and asks for them to be delivered. func (s *Server) Follows(u Upgrader) error { - if _, err := s.channel.QueueDeclare(UpgradeQueue, true, false, false, false, nil); err != nil { - return fmt.Errorf("cannot declare the %s queue: %w", UpgradeQueue, err) - } - if err := s.channel.QueueBind(UpgradeQueue, KeyModuleUpgraded, EventsExchange, false, nil); err != nil { - return fmt.Errorf("cannot bind %s to %s/%s: %w", UpgradeQueue, EventsExchange, KeyModuleUpgraded, err) + if err := s.inbound.Also(KindModuleMoved); err != nil { + return err } s.upgrader = u return nil } -// Answers binds the queue a catalogue's catch-up request arrives on. -// -// **Not bound unless something is listening**, for the same reason upgrades are not: a durable -// queue with no consumer fills quietly and the first symptom is a broker out of disk. +// Answers says what to do about a catalogue's catch-up request, and asks for them to be delivered. func (s *Server) Answers(r Replayer) error { - if _, err := s.channel.QueueDeclare(CatchUpQueue, true, false, false, false, nil); err != nil { - return fmt.Errorf("cannot declare the %s queue: %w", CatchUpQueue, err) - } - if err := s.channel.QueueBind(CatchUpQueue, KeyCatchingUp, EventsExchange, false, nil); err != nil { - return fmt.Errorf("cannot bind %s to %s/%s: %w", CatchUpQueue, EventsExchange, KeyCatchingUp, err) + if err := s.inbound.Also(KindCatchUp); err != nil { + return err } s.replayer = r return nil } -// Connect opens the control plane's own connection to the broker. +// Connect opens the controller's own connection to the bus. // // On the port MESH_BROKER_AMQP_PORT names when the node's settings moved the broker (novox/hq // 04-ISSUES/102) — the URL is genesis's, sealed, and its port is the one thing in it the node may @@ -164,7 +134,7 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { conn, err := amqp.Dial(url) if err != nil { - // Not quoted back: the URL carries the control plane's own broker password. + // Not quoted back: the URL carries the controller's own bus password. return nil, fmt.Errorf("cannot reach the broker named in %s: %w", AMQPVar, err) } channel, err := conn.Channel() @@ -173,17 +143,17 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { return nil, err } - // Declared here rather than assumed. The control plane is the only thing that may create - // them — a node's account can write to this exchange and read its own queue, and configure - // nothing else, so a node arriving before the control plane has ever run finds nothing and - // says so, rather than quietly creating a topology nobody designed. + // Declared here rather than assumed. The controller is the only thing that may create them — a + // node's account can write to this exchange and read its own queue, and configure nothing + // else, so a node arriving before the controller has ever run finds nothing and says so, + // rather than quietly creating a topology nobody designed. if err := channel.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil { conn.Close() return nil, fmt.Errorf("cannot declare the %s exchange: %w", Exchange, err) } - // The events exchange too. The control plane is not the only publisher on it — modules - // announce onto it with their own accounts — but it is the only thing permitted to create it, - // for the same reason it is the only thing permitted to create the direct one. + // The events exchange too. The controller is not the only publisher on it — modules announce + // onto it with their own accounts — but it is the only thing permitted to create it, for the + // same reason it is the only thing permitted to create the direct one. if err := channel.ExchangeDeclare(EventsExchange, "topic", true, false, false, false, nil); err != nil { conn.Close() return nil, fmt.Errorf("cannot declare the %s exchange: %w", EventsExchange, err) @@ -192,7 +162,7 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { conn.Close() return nil, fmt.Errorf("cannot declare the %s queue: %w", ControlQueue, err) } - // Every key a node may publish. Binding one and forgetting another is a message the broker + // Every key a node may publish. Binding one and forgetting another is a message the bus // accepts, finds no queue for, and drops — the publisher sees success and the consumer sees // nothing. That is exactly what happened to reports: `report` was left unbound while `enrol` // worked, so nodes announced what they had applied into a void for an afternoon. @@ -203,14 +173,24 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { } } - return &Server{conn: conn, channel: channel, enroller: enroller, listener: listener, - log: log.New(os.Stdout, "", log.LstdFlags)}, nil + return &Server{ + inbound: Current(conn, channel), + bus: OverCurrent{Channel: channel}, + conn: conn, + channel: channel, + enroller: enroller, + listener: listener, + log: log.New(os.Stdout, "", log.LstdFlags), + }, nil } -// Channel is the control plane's channel, for sending declarations. +// Channel is the controller's channel, for the command line's own publishing. func (s *Server) Channel() *amqp.Channel { return s.channel } func (s *Server) Close() { + if s.inbound != nil { + s.inbound.Close() + } if s.channel != nil { _ = s.channel.Close() } @@ -219,131 +199,120 @@ func (s *Server) Close() { } } -// Serve consumes until the context ends. -// -// One consumer, deliberately: with two, the broker would round-robin between them and each would -// receive half of what it expects — a fault this project has already had, between a module's -// daemon and its capability server. +// Serve acts on what arrives until the context ends. func (s *Server) Serve(ctx context.Context) error { - // A bounded prefetch rather than one. The loop still takes messages one at a time; what the - // prefetch buys is that a message the store could not take can be held while the loop goes on - // to the next, instead of every enrolment waiting behind it (novox/hq issue 083). Anything held - // goes back to the broker if the control plane stops, because nothing held is acknowledged. - if err := s.channel.Qos(Prefetch, 0, false); err != nil { - return err - } - - deliveries, err := s.channel.ConsumeWithContext(ctx, ControlQueue, "control-plane", - false, false, false, false, nil) - if err != nil { - return err - } - - // The upgrade queue, when something is listening for them. A second queue rather than a - // second consumer on the first: two consumers on one queue split its messages between them, - // which is the fault the comment above exists about. Two queues share nothing. - var upgrades <-chan amqp.Delivery + s.log.Printf("consuming what nodes say: %s, %s, %s, %s", + KindEnrolment, KindReport, KindHeartbeat, KindBuilt) if s.upgrader != nil { - upgrades, err = s.channel.ConsumeWithContext(ctx, UpgradeQueue, "control-plane-upgrades", - false, false, false, false, nil) - if err != nil { - return err - } + s.log.Printf("following %s", KindModuleMoved) } - - // Its own queue and its own consumer, for the reason above: two consumers on one queue split - // its messages, and a catch-up request going to whichever half was not listening is a gap that - // looks like a working mesh. - var catchups <-chan amqp.Delivery if s.replayer != nil { - catchups, err = s.channel.ConsumeWithContext(ctx, CatchUpQueue, "control-plane-catchup", - false, false, false, false, nil) - if err != nil { - return err - } - } - - closed := s.conn.NotifyClose(make(chan *amqp.Error, 1)) - s.log.Printf("consuming %s, bound to %s/{%s,%s,%s,%s}", - ControlQueue, Exchange, KeyEnrol, KeyReport, KeyAlive, KeyBuilt) - if s.upgrader != nil { - s.log.Printf("consuming %s, bound to %s/%s", UpgradeQueue, EventsExchange, KeyModuleUpgraded) - } - - again := s.again - if again == 0 { - again = TryAgainAfter - } - ticker := time.NewTicker(again) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - if ctx.Err() != nil { - return nil - } - s.retryHeld(ctx) - case delivery, ok := <-catchups: - if !ok { - if catchups != nil { - return errors.New("the broker stopped delivering catch-up requests") - } - continue - } - s.catchingUp(ctx, delivery) - case delivery, ok := <-upgrades: - // A nil channel blocks for ever, so this case simply never fires when nothing is - // listening for upgrades. Closed is different, and means the broker stopped. - if !ok { - if upgrades != nil { - return errors.New("the broker stopped delivering upgrades") - } - continue - } - s.upgraded(ctx, delivery) - case reason := <-closed: - // Said rather than returned quietly. A control plane whose broker connection dropped - // is a mesh where nothing can be told anything, and the reason is the first thing - // anybody will want. - return fmt.Errorf("the broker connection closed: %v", reason) - case delivery, ok := <-deliveries: - if !ok { - return errors.New("the broker stopped delivering") - } - s.handle(ctx, delivery) - } + s.log.Printf("answering %s", KindCatchUp) } + return s.inbound.Receive(ctx, s.act) } -func (s *Server) handle(ctx context.Context, delivery amqp.Delivery) { - switch delivery.RoutingKey { - case KeyEnrol: - s.handleEnrol(ctx, delivery) - case KeyReport: - s.handleReport(ctx, delivery) - case KeyAlive: - s.handleAlive(delivery) - case KeyBuilt: - s.handleBuilt(ctx, delivery) +// act is one message, whichever bus it came over. +func (s *Server) act(ctx context.Context, m Control) { + switch m.Kind() { + case KindEnrolment: + s.enrolling(ctx, m) + case KindReport: + s.reported(ctx, m) + case KindHeartbeat: + s.heartbeat(m) + case KindBuilt: + s.wasBuilt(ctx, m) + case KindModuleMoved: + s.moved(ctx, m) + case KindCatchUp: + s.catchingUp(ctx, m) default: - // Rejected without requeue: a message nothing understands will not be understood on the - // next attempt either, and requeuing it would spin. - s.log.Printf("refusing a message with routing key %q", delivery.RoutingKey) - _ = delivery.Reject(false) + // Dropped: a message nothing understands will not be understood on the next attempt + // either, and asking for it again would spin. + s.log.Printf("refusing a message the mesh has no name for: %q", m.Kind()) + _ = m.Drop() } } -// handleAlive records that a node was heard from, and nothing else. +// decide asks the window what to do about one message, and holds it when that is the answer — +// because holding is the one verdict that means the same thing everywhere. // -// Deliberately silent: a node saying it is there every minute would fill the log with the -// ordinary case, and a log where the ordinary case is loud is a log nobody reads. -func (s *Server) handleAlive(delivery amqp.Delivery) { +// The other three come back, because "settled without acting" is a build result dropped and an +// upgrade taken, and only the handler knows which its message is. Hold means the message is the +// bus's problem now and **the caller must not settle it**; that is also what a cancelled context +// gets, because shutting down is not an answer about a message (novox/hq issue 083, on review). +// +// declaredIn is the declaration this message is about, empty when it is about none; outstanding is +// what the mesh last sent that node, empty when it has sent none or could not be asked. +func (s *Server) decide(ctx context.Context, m Control, what, declaredIn, outstanding string, + err error) Verdict { + + if ctx.Err() != nil { + return Hold + } + window := StoreWindow{GiveUpAfter: s.giveUpAfter()} + switch v := window.Decide(err, declaredIn, outstanding, m.HeldFor()); v { + case Hold: + // Said once, on the first hold. Every redelivery saying it again would fill the log with + // one store restart. + first := m.HeldFor() == 0 + if held := m.Hold(RedeliverAfter(m.HeldFor())); held != nil { + s.log.Printf("LOST %s: it could not be held for the store (%v): %v", what, held, err) + return GiveUp + } + if first { + s.log.Printf("could not keep %s yet; holding it to try again: %v", what, err) + } + return Hold + case Stale: + s.log.Printf("set aside %s: the mesh has moved past that declaration", what) + return Stale + case GiveUp: + s.log.Printf("LOST %s: the store has not come back in %s: %v", what, s.giveUpAfter(), err) + return GiveUp + default: + return v + } +} + +func (s *Server) giveUpAfter() time.Duration { + if s.giveUp == 0 { + return GiveUpAfter + } + return s.giveUp +} + +// outstanding is the digest of the declaration the mesh last sent a node. +// +// Nothing is guessed when it cannot be answered: a listener that keeps no record of what was sent +// gives a report nothing to be stale against, and a store that cannot be read will hold the report +// anyway, so there is nothing for the check to decide. +func (s *Server) outstanding(ctx context.Context, node string) string { + if node == "" { + return "" + } + asks, ok := s.listener.(Outstanding) + if !ok { + return "" + } + digest, err := asks.Outstanding(ctx, node) + if err != nil { + return "" + } + return digest +} + +// heartbeat records that a node was heard from, and nothing else. +// +// Deliberately silent: a node saying it is there every minute would fill the log with the ordinary +// case, and a log where the ordinary case is loud is a log nobody reads. Not held for the store +// either — the next heartbeat is a minute away, and a heartbeat kept for two minutes to be written +// late says nothing the one after it will not say better. +func (s *Server) heartbeat(m Control) { var alive Alive - if err := json.Unmarshal(delivery.Body, &alive); err != nil || alive.Node == "" { - _ = delivery.Reject(false) + if err := json.Unmarshal(m.Body(), &alive); err != nil || alive.Node == "" { + _ = m.Drop() return } if s.listener != nil { @@ -351,34 +320,49 @@ func (s *Server) handleAlive(delivery amqp.Delivery) { s.log.Printf("could not record that %s is here: %v", alive.Node, err) } } - _ = delivery.Ack(false) + _ = m.Took() } -// handleReport records what a node says it did. +// reported records what a node says it did. // // A node states; nothing here writes anything the node claimed about itself beyond that it was // heard from. What it applied is its own account of its own machine, and the mesh keeps the last // one as a copy for recovery rather than as a source (novox/hq 09-the-node-lifecycle). -func (s *Server) handleReport(ctx context.Context, delivery amqp.Delivery) { +func (s *Server) reported(ctx context.Context, m Control) { var report Report - if err := json.Unmarshal(delivery.Body, &report); err != nil { + if err := json.Unmarshal(m.Body(), &report); err != nil { s.log.Printf("a report could not be read: %v", err) - _ = delivery.Reject(false) + _ = m.Drop() return } - // A node's newer report supersedes one of its older reports still held: the older is its - // past, and recorded after the newer it would overwrite what the node is doing now. - subject := "report " + report.Node - s.supersede(subject, delivery) + m.About("report " + report.Node) + what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared) + if s.listener != nil { - if err := s.listener.Heard(context.Background(), report); err != nil { - // Held, not acknowledged, while the store cannot take it: the node reports an apply - // once, and a report lost here is a node the mesh never hears from again — the store + // **Asked before the store, not after** (design 25 §3). A report about a declaration the + // mesh has moved past would otherwise wait out a restarting store to be written and then + // overwrite what the node is doing now — and on the bus being built, where the holding is + // the server's, it comes back after the newer was applied whatever the controller does. + outstanding := s.outstanding(ctx, report.Node) + declaredIn := staleAgainst(report) + if Superseded(declaredIn, outstanding) { + s.log.Printf("set aside %s: the mesh has moved past that declaration", what) + _ = m.Took() + return + } + + err := s.listener.Heard(context.Background(), report) + switch s.decide(ctx, m, what, declaredIn, outstanding, err) { + case Hold: + // Held, not settled, while the store cannot take it: the node reports an apply once, + // and a report lost here is a node the mesh never hears from again — the store // restarting under the adoption that node just applied lost exactly that (issue 082). - what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared) - if s.tryLater(ctx, delivery, subject, what, err, s.handle) { - return - } + return + case Stale, GiveUp: + _ = m.Took() + return + } + if err != nil { // Said rather than swallowed. A report the mesh heard and failed to write down is a // node whose recovery copy is silently older than it looks. s.log.Printf("could not record %s's report: %v", report.Node, err) @@ -393,124 +377,49 @@ func (s *Server) handleReport(ctx context.Context, delivery amqp.Delivery) { default: s.log.Printf("%s applied %d resource(s)", report.Node, len(report.Applied)) } - s.settled(subject, delivery) - _ = delivery.Ack(false) + _ = m.Took() } -// tryLater holds a message the store could not take right now, to be tried again on the ticker, -// and says whether it did (novox/hq issues 082, 083). +// staleAgainst is the declaration a report may be judged stale against, and it is empty for a +// report that is not only an account of an apply. // -// "Right now" is the store unreachable or restarting — ErrTryAgain from a listener, or an error the -// inventory reads as an outage. Anything else is an answer, and is left to the caller to settle. -// Held means unacknowledged and set aside: the loop goes on to the next message, so an enrolment a -// host is waiting on is answered while a report waits for the store. One message is held at most -// giveUpAfter; past it, it is let go with a line saying it was lost, and the caller settles it. -func (s *Server) tryLater(ctx context.Context, delivery amqp.Delivery, subject, what string, err error, - retry func(context.Context, amqp.Delivery)) bool { - // Shutting down: nothing is settled. Unsettled, the broker hands the message to whatever - // consumes next — a cancelled context is not an answer about the message (issue 083, review). - if ctx.Err() != nil { - return true +// **A report carries two different things, and only one of them is about a declaration.** What the +// node applied is; what the machine *is* — the tunnel it took over, the ports its own bundle holds, +// what an adopted node found and is keeping, a node moving its overlay key — is not. Those reach +// the mesh on a report because a report is the message a node sends, and nowhere else: a rekey +// dropped as stale is a node whose overlay key never moves, and no retry is coming, because the +// node said it once. +// +// So staleness is asked only of a report that is purely an apply's account. The rest is acted on +// whenever it arrives, which is the behaviour the mesh has had all along. +func staleAgainst(report Report) string { + if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 || + report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 { + return "" } - if !errors.Is(err, ErrTryAgain) && !inventory.Unreachable(err) { - return false - } - if s.parked == nil { - s.parked = map[string]*held{} - } - h, ok := s.parked[subject] - if !ok || h.delivery.DeliveryTag != delivery.DeliveryTag { - // Held no further than the prefetch leaves room: past it, the broker would hand the loop - // nothing new — enrolments included — until something held was let go. - if !ok && len(s.parked) >= Prefetch-PrefetchHeadroom { - s.log.Printf("LOST %s: %d messages are already held for the store, and holding more "+ - "would stop the queue: %v", what, len(s.parked), err) - return false - } - h = &held{delivery: delivery, retry: retry, what: what, first: time.Now()} - s.parked[subject] = h - s.log.Printf("could not keep %s yet; holding it to try again: %v", what, err) - return true - } - if time.Since(h.first) >= s.giveUpAfter() { - delete(s.parked, subject) - s.log.Printf("LOST %s: the store has not come back in %s: %v", what, s.giveUpAfter(), err) - return false - } - return true + return report.Declared } -// supersede drops a message held for a subject when a newer one for it arrives: the older is -// acknowledged, because acting on it after the newer would undo the newer. -func (s *Server) supersede(subject string, newer amqp.Delivery) { - h, ok := s.parked[subject] - if !ok || h.delivery.DeliveryTag == newer.DeliveryTag { - return - } - delete(s.parked, subject) - s.log.Printf("set aside %s: a newer one arrived", h.what) - _ = h.delivery.Ack(false) -} - -// settled forgets a message once it has been handled either way. -func (s *Server) settled(subject string, delivery amqp.Delivery) { - if h, ok := s.parked[subject]; ok && h.delivery.DeliveryTag == delivery.DeliveryTag { - delete(s.parked, subject) - } -} - -// digest names a message by its content. -func digest(body []byte) string { - sum := sha256.Sum256(body) - return hex.EncodeToString(sum[:]) -} - -// retryHeld tries every held message again. Each handler holds it again, settles it, or lets it -// go past the bound. -func (s *Server) retryHeld(ctx context.Context) { - for _, h := range s.snapshot() { - if ctx.Err() != nil { - return - } - h.retry(ctx, h.delivery) - } -} - -func (s *Server) snapshot() []*held { - out := make([]*held, 0, len(s.parked)) - for _, h := range s.parked { - out = append(out, h) - } - return out -} - -func (s *Server) giveUpAfter() time.Duration { - if s.giveUp == 0 { - return GiveUpAfter - } - return s.giveUp -} - -func (s *Server) handleEnrol(ctx context.Context, delivery amqp.Delivery) { +func (s *Server) enrolling(ctx context.Context, m Control) { reply := EnrolReply{Refusal: "that token cannot be used"} var request EnrolRequest - if err := json.Unmarshal(delivery.Body, &request); err != nil { + if err := json.Unmarshal(m.Body(), &request); err != nil { s.log.Printf("an enrolment request could not be read: %v", err) } else { - request.Redelivered = delivery.Redelivered + request.Redelivered = m.Redelivered() accepted, err := s.enroller.Enrol(ctx, request) switch { case errors.Is(err, ErrTryAgain): // Not a refusal: nothing was spent, and the same request asked again will be - // answered. Replied at once rather than held, so the node — which is waiting on - // this answer — decides when to ask, and the queue behind it moves (issue 083). + // answered. Replied at once rather than held, so the node — which is waiting on this + // answer — decides when to ask, and the queue behind it moves (issue 083). reply = EnrolReply{TryAgain: true, Refusal: "the mesh cannot answer right now; ask again"} s.log.Printf("asked %q to enrol again shortly: %v", request.Node, err) case err != nil && request.Redelivered: - // Said as what it most likely is: the broker handed this request over again after - // the control plane stopped mid-answer, and an enrolment already spent is not - // finished a second time. The node may need a new token. + // Said as what it most likely is: the bus handed this request over again after the + // controller stopped mid-answer, and an enrolment already spent is not finished a + // second time. The node may need a new token. s.log.Printf("refusing a redelivered enrolment for %q — it may have finished before "+ "the control plane stopped, and if the node did not get its answer it needs a new "+ "token: %v", request.Node, err) @@ -524,169 +433,160 @@ func (s *Server) handleEnrol(ctx context.Context, delivery amqp.Delivery) { } } - s.reply(ctx, delivery, reply) - - // Acknowledged after the reply is sent, so a control plane that dies mid-answer leaves the - // request on the broker rather than having consumed it silently. Asked again by the same - // presenter, an enrolment finishes: the token is held for its key and spent last (issue 083). - _ = delivery.Ack(false) -} - -func (s *Server) reply(ctx context.Context, delivery amqp.Delivery, reply EnrolReply) { - if delivery.ReplyTo == "" { - s.log.Print("an enrolment request named no reply queue, so nothing can be told the answer") - return - } - body, err := json.Marshal(reply) - if err != nil { + if body, err := json.Marshal(reply); err != nil { s.log.Printf("cannot encode a reply: %v", err) - return + } else { + answer, cancel := context.WithTimeout(ctx, 10*time.Second) + if err := m.Answer(answer, body); err != nil { + s.log.Printf("cannot answer an enrolment: %v", err) + } + cancel() } - timeout, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if err := s.channel.PublishWithContext(timeout, "", delivery.ReplyTo, false, false, - amqp.Publishing{ - ContentType: "application/json", - CorrelationId: delivery.CorrelationId, - Body: body, - }); err != nil { - s.log.Printf("cannot reply to %s: %v", delivery.ReplyTo, err) - } + // Settled after the answer is sent, so a controller that dies mid-answer leaves the request on + // the bus rather than having consumed it silently. Asked again by the same presenter, an + // enrolment finishes: the token is held for its key and spent last (issue 083). + _ = m.Took() } -// handleBuilt keeps what a builder said, whichever way it went. +// wasBuilt keeps what a builder said, whichever way it went. // // This is for results nobody was waiting for. A build asked for with `build` is answered directly // to the asker; one triggered any other way is published here, and without this it would be // reported into the void — which is the same as not reporting it. -func (s *Server) handleBuilt(ctx context.Context, delivery amqp.Delivery) { +func (s *Server) wasBuilt(ctx context.Context, m Control) { var result BuildResult - if err := json.Unmarshal(delivery.Body, &result); err != nil { + if err := json.Unmarshal(m.Body(), &result); err != nil { s.log.Printf("a build result could not be read: %v", err) - _ = delivery.Reject(false) + _ = m.Drop() return } if s.recorder == nil { - // Nothing to keep it in. Rejected rather than dropped silently, so the broker's own - // counters show something arriving that nothing handles. + // Nothing to keep it in. Dropped rather than swallowed, so the bus's own counters show + // something arriving that nothing handles. s.log.Printf("a build result arrived and this control plane keeps none") - _ = delivery.Reject(false) + _ = m.Drop() return } // Each build result its own subject: none supersedes another, and recording one twice is // harmless — the build is kept by its id. - subject := "build " + digest(delivery.Body) - s.supersede(subject, delivery) - if err := s.recorder.Built(ctx, result); err != nil { + m.About("build " + digest(m.Body())) + err := s.recorder.Built(ctx, result) + switch s.decide(ctx, m, fmt.Sprintf("a build result from %s", result.On), "", "", err) { + case Hold: // Held while the store cannot take it: a build result lost here is never announced (083). - if s.tryLater(ctx, delivery, subject, fmt.Sprintf("a build result from %s", result.On), err, s.handle) { - return - } + return + case Stale, GiveUp: + _ = m.Drop() + return + } + if err != nil { s.log.Printf("cannot keep a build result from %s: %v", result.On, err) - s.settled(subject, delivery) - _ = delivery.Reject(false) + _ = m.Drop() return } - s.settled(subject, delivery) switch { case result.Failed != "": s.log.Printf("%s could not build %s", result.On, result.Repository) default: s.log.Printf("%s built %s from %s", result.On, result.Repository, result.Commit) } - _ = delivery.Ack(false) + _ = m.Took() } // catchingUp answers a catalogue that has just started and may have missed builds. // -// Acknowledged after the work. A replay that fails for a reason other than the store is not one -// that succeeds by being handed the same request again, so that is acknowledged and said; but a -// store that could not be read right now is held and asked again, bounded, rather than the -// request lost until the catalogue next restarts (issue 083). One request stands for all: a newer -// one supersedes one still held. -func (s *Server) catchingUp(ctx context.Context, delivery amqp.Delivery) { - const subject = "catch-up" - s.supersede(subject, delivery) - holding := false - defer func() { - if !holding { - s.settled(subject, delivery) - _ = delivery.Ack(false) - } - }() +// Settled after the work. A replay that fails for a reason other than the store is not one that +// succeeds by being handed the same request again, so that is settled and said; but a store that +// could not be read right now is held and asked again, bounded, rather than the request lost until +// the catalogue next restarts (issue 083). One request stands for all: a newer one supersedes one +// still held. +func (s *Server) catchingUp(ctx context.Context, m Control) { + m.About("catch-up") if s.replayer == nil { s.log.Printf("a catalogue asked to catch up and this control plane has nothing to replay") + _ = m.Took() return } announcements, err := s.replayer.Announceable(ctx) + switch s.decide(ctx, m, "a catalogue's request to catch up", "", "", err) { + case Hold: + return + case Stale, GiveUp: + _ = m.Took() + return + } if err != nil { - if s.tryLater(ctx, delivery, subject, "a catalogue's request to catch up", err, s.catchingUp) { - holding = true - return - } s.log.Printf("a catalogue asked to catch up and the mesh could not read its builds: %v", err) + _ = m.Took() return } sent := 0 for _, a := range announcements { a.Replay = true - if err := EmitEvent(ctx, OverCurrent{Channel: s.channel}, KeyModuleBuilt, "control-plane", "", a); err != nil { + if err := EmitEvent(ctx, s.bus, KeyModuleBuilt, "control-plane", "", a); err != nil { // Said and abandoned rather than retried: the catalogue asks again every time it // starts, and half a graph delivered twice is no better than half delivered once. s.log.Printf("replaying %s at %s failed, and the rest is abandoned: %v", a.Module, short(a.Commit), err) + _ = m.Took() return } sent++ } s.log.Printf("a catalogue asked to catch up; re-announced %d build(s)", sent) + _ = m.Took() } -// upgraded hands one announcement to whatever is following them. +// moved hands one announcement to whatever is following them. // -// **Acknowledged whatever happens, but one thing.** A failure here is usually the control plane -// being unable to act on an upgrade — a machine that cannot be resolved, a broker that will not -// take a declaration — and none of those get better by being handed the same message again. The -// one exception is the upgrader saying the store could not be read for the moment (ErrTryAgain): -// that is held and asked again, bounded (novox/hq issue 083). Only the upgrader's word counts -// here, not an error that merely looks like an outage — a push that timed out on the second -// machine is not asked again, or the first would be pushed every few seconds for two minutes. -// A newer move of the same module supersedes one still held. -func (s *Server) upgraded(ctx context.Context, delivery amqp.Delivery) { +// **Settled whatever happens, but one thing.** A failure here is usually the controller being +// unable to act on an upgrade — a machine that cannot be resolved, a bus that will not take a +// declaration — and none of those get better by being handed the same message again. The one +// exception is the upgrader saying the store could not be read for the moment (ErrTryAgain): that +// is held and asked again, bounded (novox/hq issue 083). Only the upgrader's word counts here, not +// an error that merely looks like an outage — a push that timed out on the second machine is not +// asked again, or the first would be pushed every few seconds for two minutes. A newer move of the +// same module supersedes one still held. +func (s *Server) moved(ctx context.Context, m Control) { var u Upgraded - _ = json.Unmarshal(delivery.Body, &u) - subject := "upgrade " + u.Module - s.supersede(subject, delivery) - holding := false - defer func() { - if !holding { - s.settled(subject, delivery) - _ = delivery.Ack(false) - } - }() - if err := json.Unmarshal(delivery.Body, &u); err != nil { + if err := json.Unmarshal(m.Body(), &u); err != nil { s.log.Printf("an upgrade announcement could not be read: %v", err) + _ = m.Took() return } + m.About("upgrade " + u.Module) if u.Module == "" { s.log.Printf("an upgrade announcement named no module; ignored") + _ = m.Took() return } - if err := s.upgrader.Upgraded(ctx, u); err != nil { - // Shutting down is not an answer about the announcement: left for the broker. - if ctx.Err() != nil { - holding = true - return - } - if errors.Is(err, ErrTryAgain) && - s.tryLater(ctx, delivery, subject, fmt.Sprintf("%s's move to %s", u.Module, short(u.Commit)), err, s.upgraded) { - holding = true - return - } + err := s.upgrader.Upgraded(ctx, u) + // Only "not now" is worth holding. Anything else is an answer, and the window would read a + // timed-out push as an outage and ask for it again. + holdable := err + if !errors.Is(err, ErrTryAgain) { + holdable = nil + } + what := fmt.Sprintf("%s's move to %s", u.Module, short(u.Commit)) + switch s.decide(ctx, m, what, "", "", holdable) { + case Hold: + return + case Stale, GiveUp: + _ = m.Took() + return + } + if err != nil { s.log.Printf("%s moved to %s and the mesh could not act on it: %v", u.Module, short(u.Commit), err) } + _ = m.Took() +} + +// digest names a message by its content. +func digest(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) } // short is a commit as people read it. diff --git a/internal/link/stale_report_test.go b/internal/link/stale_report_test.go new file mode 100644 index 0000000..2e0b949 --- /dev/null +++ b/internal/link/stale_report_test.go @@ -0,0 +1,135 @@ +package link + +import ( + "context" + "errors" + "testing" +) + +// Supersession as a check rather than a memory (design 25 §3). +// +// Holding a message in memory let the controller drop an older report when a newer one for the +// same node arrived. On the bus being built the message belongs to the server and comes back +// whatever happened meanwhile — so the older report is redelivered *after* the newer was applied, +// and acting on it would undo the newer. +// +// The answer was already in the message: a report carries the digest of the declaration it is +// about, so "is this the past?" is a question the message answers. + +// sentAndHeard records reports and knows what was last sent, which is the pair the check needs. +type sentAndHeard struct { + sent string + heard []Report + err error +} + +func (s *sentAndHeard) Heard(_ context.Context, r Report) error { + if s.err != nil { + return s.err + } + s.heard = append(s.heard, r) + return nil +} + +func (s *sentAndHeard) Outstanding(context.Context, string) (string, error) { return s.sent, nil } + +// A report about a declaration the mesh has moved past is settled and not acted on. Settled rather +// than dropped, because there is nothing wrong with the message — it is simply the past, and +// redelivering it for ever is worse than letting it go. +func TestAReportAboutASupersededDeclarationIsNotActedOn(t *testing.T) { + store := &sentAndHeard{sent: "d2"} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) + + if len(store.heard) != 0 { + t.Fatalf("a report about a superseded declaration was acted on: %+v", store.heard) + } + if !to.acked { + t.Fatalf("a superseded report was not settled, so it comes back for ever: %+v", *to) + } +} + +// The report about the declaration that *is* outstanding is acted on, and so is one from a node +// the mesh has no digest for — an older host that says nothing about which declaration it applied +// has nothing to be judged against, and refusing it would silence every node built before reports +// carried the digest. +func TestAReportAboutTheOutstandingDeclarationIsActedOn(t *testing.T) { + for _, c := range []struct{ what, sent, declared string }{ + {"the one outstanding", "d2", "d2"}, + {"a report that says nothing about which", "d2", ""}, + {"a node nothing was ever sent", "", "d1"}, + } { + store := &sentAndHeard{sent: c.sent} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", c.declared))) + if len(store.heard) != 1 || !to.acked { + t.Errorf("%s: was not acted on and acknowledged: heard %+v, settled %+v", + c.what, store.heard, *to) + } + } +} + +// **Staleness is decided before the store is waited on**, not after: a redelivery that lost its +// race is not worth holding a slot in the window that a current message needs. +func TestASupersededReportIsNotHeldForTheStore(t *testing.T) { + store := &sentAndHeard{sent: "d2", err: errors.Join(ErrTryAgain, errors.New("starting up"))} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) + if !to.acked || len(in.held) != 0 { + t.Fatalf("a superseded report waited for the store: %+v, %d held", *to, len(in.held)) + } +} + +// **Half of a report is not about a declaration, and that half is never stale.** +// +// What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted +// node found and is keeping, a node moving its overlay key — reaches the mesh on a report and +// nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is +// coming, because the node said it once. So a report carrying any of these is acted on whenever it +// arrives, however far the mesh has moved on. +func TestAReportCarryingWhatOnlyTheNodeKnowsIsActedOnHoweverOldItIs(t *testing.T) { + for _, c := range []struct { + what string + report Report + }{ + {"a rekey", Report{Node: "anchor", Declared: "d1", + Rekey: &Rekey{Previous: "k1", OverlayKey: "k2"}}}, + {"the tunnel it carried", Report{Node: "anchor", Declared: "d1", + Tunnel: &CarriedTunnel{Interface: "wg0", State: "taken"}}}, + {"what an adopted node holds", Report{Node: "anchor", Declared: "d1", + Held: []Held{{ID: "conf", Module: "web", Kind: "file"}}}}, + {"the firewall it found", Report{Node: "anchor", Declared: "d1", Firewall: "ufw"}}, + {"what is reachable on it", Report{Node: "anchor", Declared: "d1", + Reachable: []Reach{{Protocol: "tcp", Port: 443}}}}, + {"the ports its own bundle holds", Report{Node: "anchor", Declared: "d1", + Carried: []int{5432}}}, + } { + store := &sentAndHeard{sent: "d9"} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, c.report)) + if len(store.heard) != 1 { + t.Errorf("%s was set aside as stale, and the mesh will never hear it again: %+v", + c.what, *to) + } + } +} + +// A heartbeat is not held for the store: the next one is a minute away, and one kept for two +// minutes to be written late says nothing the one after it will not say better. +func TestAHeartbeatIsNotHeldForTheStore(t *testing.T) { + s, in := serving() + s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))} + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindHeartbeat, Alive{Node: "anchor"})) + if !to.acked || len(in.held) != 0 { + t.Fatalf("a heartbeat was held for the store: %+v, %d held", *to, len(in.held)) + } +} diff --git a/internal/link/store_window_test.go b/internal/link/store_window_test.go index f0fcd2d..80b6a4b 100644 --- a/internal/link/store_window_test.go +++ b/internal/link/store_window_test.go @@ -2,15 +2,11 @@ package link import ( "context" - "encoding/json" "errors" "fmt" - "io" - "log" "testing" "github.com/jackc/pgx/v5/pgconn" - amqp "github.com/rabbitmq/amqp091-go" ) // What a store restarting under an adoption answers with (novox/hq issues 082, 083). @@ -28,29 +24,6 @@ type replaysWith struct{ err error } func (r replaysWith) Announceable(context.Context) ([]Announcement, error) { return nil, r.err } -type settledAs struct{ acked, nacked, requeued, rejected bool } - -func (a *settledAs) Ack(uint64, bool) error { a.acked = true; return nil } -func (a *settledAs) Nack(_ uint64, _ bool, requeue bool) error { - a.nacked, a.requeued = true, requeue - return nil -} -func (a *settledAs) Reject(uint64, bool) error { a.rejected = true; return nil } - -func a(t *testing.T, to *settledAs, key string, v any) amqp.Delivery { - t.Helper() - body, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - tag++ - return amqp.Delivery{Acknowledger: to, RoutingKey: key, Body: body, DeliveryTag: tag} -} - -func quietServer() *Server { return &Server{log: log.New(io.Discard, "", 0)} } - -func (a *settledAs) held() bool { return !a.acked && !a.nacked && !a.rejected } - // A build result the store could not take right now is handed back; one it refused is rejected, // as before; one it kept is acknowledged. func TestABuildResultWaitsOutARestartingStore(t *testing.T) { @@ -58,16 +31,16 @@ func TestABuildResultWaitsOutARestartingStore(t *testing.T) { for _, c := range []struct { what string err error - want func(*settledAs) bool + want func(*settled) bool }{ - {"restarting", restarting, func(s *settledAs) bool { return s.held() }}, - {"refused", errors.New("no such module"), func(s *settledAs) bool { return s.rejected && !s.nacked }}, - {"kept", nil, func(s *settledAs) bool { return s.acked && !s.nacked }}, + {"restarting", restarting, func(s *settled) bool { return s.unsettled() }}, + {"refused", errors.New("no such module"), func(s *settled) bool { return s.rejected && !s.nacked }}, + {"kept", nil, func(s *settled) bool { return s.acked && !s.nacked }}, } { - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: c.err} - to := &settledAs{} - s.handleBuilt(context.Background(), a(t, to, KeyBuilt, built)) + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindBuilt, built)) if !c.want(to) { t.Errorf("%s: a build result was settled as %+v", c.what, *to) } @@ -81,17 +54,17 @@ func TestAnUpgradeWaitsOutARestartingStoreAndNothingElse(t *testing.T) { for _, c := range []struct { what string err error - want func(*settledAs) bool + want func(*settled) bool }{ - {"the store away, said by the upgrader", errors.Join(ErrTryAgain, restarting), func(s *settledAs) bool { return s.held() }}, - {"a push that timed out", context.DeadlineExceeded, func(s *settledAs) bool { return s.acked && !s.nacked }}, - {"cannot act", errors.New("anchor cannot be resolved"), func(s *settledAs) bool { return s.acked && !s.nacked }}, - {"acted", nil, func(s *settledAs) bool { return s.acked && !s.nacked }}, + {"the store away, said by the upgrader", errors.Join(ErrTryAgain, restarting), func(s *settled) bool { return s.unsettled() }}, + {"a push that timed out", context.DeadlineExceeded, func(s *settled) bool { return s.acked && !s.nacked }}, + {"cannot act", errors.New("anchor cannot be resolved"), func(s *settled) bool { return s.acked && !s.nacked }}, + {"acted", nil, func(s *settled) bool { return s.acked && !s.nacked }}, } { - s := quietServer() + s, in := serving() s.upgrader = upgradesWith{err: c.err} - to := &settledAs{} - s.upgraded(context.Background(), a(t, to, "upgraded", moved)) + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindModuleMoved, moved)) if !c.want(to) { t.Errorf("%s: an upgrade was settled as %+v", c.what, *to) } @@ -104,16 +77,16 @@ func TestACatchUpWaitsOutARestartingStore(t *testing.T) { for _, c := range []struct { what string err error - want func(*settledAs) bool + want func(*settled) bool }{ - {"restarting", restarting, func(s *settledAs) bool { return s.held() }}, - {"unreadable", errors.New("a build row is malformed"), func(s *settledAs) bool { return s.acked && !s.nacked }}, - {"nothing to replay", nil, func(s *settledAs) bool { return s.acked && !s.nacked }}, + {"restarting", restarting, func(s *settled) bool { return s.unsettled() }}, + {"unreadable", errors.New("a build row is malformed"), func(s *settled) bool { return s.acked && !s.nacked }}, + {"nothing to replay", nil, func(s *settled) bool { return s.acked && !s.nacked }}, } { - s := quietServer() + s, in := serving() s.replayer = replaysWith{err: c.err} - to := &settledAs{} - s.catchingUp(context.Background(), a(t, to, "catch-up", map[string]string{})) + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindCatchUp, map[string]string{})) if !c.want(to) { t.Errorf("%s: a catch-up request was settled as %+v", c.what, *to) } @@ -121,15 +94,15 @@ func TestACatchUpWaitsOutARestartingStore(t *testing.T) { } // Shutting down is not an answer about a message: one handled with a cancelled context is left -// unsettled, for the broker to hand to whatever consumes next (issue 083, review). -func TestAMessageHandledDuringShutdownIsLeftForTheBroker(t *testing.T) { +// unsettled, for the bus to hand to whatever consumes next (issue 083, review). +func TestAMessageHandledDuringShutdownIsLeftForTheBus(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: context.Canceled} - to := &settledAs{} - s.handleBuilt(ctx, a(t, to, KeyBuilt, BuildResult{On: "anchor", Repository: "/r", Commit: "abc"})) - if !to.held() { + to := &settled{} + s.act(ctx, in.sends(t, to, KindBuilt, BuildResult{On: "anchor", Repository: "/r", Commit: "abc"})) + if !to.unsettled() { t.Fatalf("a build result handled during shutdown was settled, and so lost: %+v", *to) } } @@ -137,45 +110,45 @@ func TestAMessageHandledDuringShutdownIsLeftForTheBroker(t *testing.T) { // Two identical build results: the newer sets the older aside rather than leaving it unsettled // for ever, holding a place in the prefetch. func TestAnIdenticalBuildResultSetsTheHeldOneAside(t *testing.T) { - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: restarting} built := BuildResult{On: "anchor", Repository: "/r", Commit: "abc"} - first, second := &settledAs{}, &settledAs{} - s.handleBuilt(context.Background(), a(t, first, KeyBuilt, built)) - s.handleBuilt(context.Background(), a(t, second, KeyBuilt, built)) - if !first.acked || !second.held() || len(s.parked) != 1 { + first, second := &settled{}, &settled{} + s.act(context.Background(), in.sends(t, first, KindBuilt, built)) + s.act(context.Background(), in.sends(t, second, KindBuilt, built)) + if !first.acked || !second.unsettled() || len(in.held) != 1 { t.Fatalf("an identical build result did not set the held one aside: first %+v second %+v, %d held", - *first, *second, len(s.parked)) + *first, *second, len(in.held)) } } // What is held stops short of the prefetch, so the loop always has room to answer an enrolment. func TestWhatIsHeldLeavesRoomInThePrefetch(t *testing.T) { - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: restarting} - var last *settledAs + var last *settled for i := 0; i < Prefetch; i++ { - last = &settledAs{} - s.handleBuilt(context.Background(), a(t, last, KeyBuilt, BuildResult{On: "anchor", Commit: fmt.Sprint(i)})) + last = &settled{} + s.act(context.Background(), in.sends(t, last, KindBuilt, BuildResult{On: "anchor", Commit: fmt.Sprint(i)})) } - if len(s.parked) != Prefetch-PrefetchHeadroom { - t.Fatalf("%d messages were held; the ceiling is %d", len(s.parked), Prefetch-PrefetchHeadroom) + if len(in.held) != Prefetch-PrefetchHeadroom { + t.Fatalf("%d messages were held; the ceiling is %d", len(in.held), Prefetch-PrefetchHeadroom) } - if last.held() { + if last.unsettled() { t.Fatalf("a message past the ceiling was held: %+v", *last) } } -// An upgrade handled during shutdown is left for the broker too — the upgrader's error is the +// An upgrade handled during shutdown is left for the bus too — the upgrader's error is the // cancelled context, which is no answer about the announcement. -func TestAnUpgradeHandledDuringShutdownIsLeftForTheBroker(t *testing.T) { +func TestAnUpgradeHandledDuringShutdownIsLeftForTheBus(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s := quietServer() + s, in := serving() s.upgrader = upgradesWith{err: context.Canceled} - to := &settledAs{} - s.upgraded(ctx, a(t, to, "upgraded", Upgraded{Module: "gitea", Commit: "abcdef0123"})) - if !to.held() { - t.Fatalf("an upgrade handled during shutdown was settled, and so lost: %+v", *to) + to := &settled{} + s.act(ctx, in.sends(t, to, KindModuleMoved, Upgraded{Module: "gitea", Commit: "abcdef0123"})) + if !to.unsettled() { + t.Fatalf("an upgrade was settled during shutdown, and so lost: %+v", *to) } } diff --git a/internal/link/window.go b/internal/link/window.go index 2b89a15..91db759 100644 --- a/internal/link/window.go +++ b/internal/link/window.go @@ -52,7 +52,7 @@ func (w StoreWindow) Decide(err error, declaredIn, outstanding string, heldFor t // **Staleness is checked before the store, not after.** A redelivery that lost its race is // not worth waiting on a store for, and asking the store first would mean a message about a // superseded declaration holding a slot in the window that a current one needs. - if declaredIn != "" && outstanding != "" && declaredIn != outstanding { + if Superseded(declaredIn, outstanding) { return Stale } if err == nil { @@ -69,6 +69,20 @@ func (w StoreWindow) Decide(err error, declaredIn, outstanding string, heldFor t return Hold } +// Superseded says a message is about a declaration the mesh has already moved past. +// +// Stated on its own because it is asked in two places for one reason: here, so the window's whole +// decision is in one pure function, and by the serving loop *before* it asks the store, because +// that is the point — a message about the past must not wait on a store, or it holds a slot in the +// window that a current message needs. +// +// Unanswerable is not stale. A message that names no declaration, and a node the mesh has never +// sent one, both give nothing to compare: the mesh acts on the message rather than guessing, which +// is also what keeps a host built before reports carried the digest from going silent. +func Superseded(declaredIn, outstanding string) bool { + return declaredIn != "" && outstanding != "" && declaredIn != outstanding +} + // RedeliverAfter is how long the server should hold a naked message before trying again. // // Backed off, and bounded. A store restarting is back in seconds; a store that is gone is not -- 2.54.0 From 88bef39952f65185f98cde6a40ac943aa8a5415c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 00:53:33 +0200 Subject: [PATCH 21/39] The consume side on NATS, and the window held by the server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The other implementation behind the seam, so the store-window guarantee now has both: one loop, one message at a time, the same window deciding. What differs is where a held message lives, and that is the whole point of the move — the AMQP side keeps an unacknowledged delivery in this process, bounded by the prefetch and lost if the controller stops; this keeps eight bytes saying when the window opened, and the message stays the server's. Checked against a running server, seven claims that reasoning cannot answer: a report is heard and leaves the work queue; one the store cannot take is naked with a delay, stays in the stream, and is recorded when the store returns; one about a superseded declaration is settled without being acted on; one the store never takes is let go once the bound passes; a heartbeat is heard and nothing is persisted; and the enrolment answer reaches the address the request carried in its payload — the test design 25 §2 asks for, so the reason for that field cannot quietly become folklore. Three things the wiring forced into the open: **The controller could not have consumed a module event.** Its permissions granted no event subject to subscribe and no ack subject on the events stream, so every announcement would have been redelivered for ever, refused by the list it already had. Both narrow: each followed subject named, not `mesh.mod.*.>`. **The controller's consumers are not derived.** It files no manifest, so its authority cannot come from a declaration that does not exist; they sit beside the mesh's own streams and are asserted the same way. No max-deliver on CONTROL — the window's bound is the controller's, and a server that dead-lettered first would discard the push the stream exists to protect. **Channels, not callbacks.** The library would run a handler on its own goroutine, and the window's bookkeeping is unlocked because the AMQP loop never had two. --- internal/broker/derived.go | 7 + internal/broker/jetstream.go | 10 +- internal/broker/nats.go | 10 + internal/broker/streams.go | 76 +++++ internal/broker/testdata/composed.conf | 4 +- internal/link/bus.go | 28 ++ internal/link/protocol.go | 14 + internal/link/receive_nats.go | 285 +++++++++++++++++++ internal/link/receive_nats_test.go | 365 +++++++++++++++++++++++++ 9 files changed, 796 insertions(+), 3 deletions(-) create mode 100644 internal/link/receive_nats.go create mode 100644 internal/link/receive_nats_test.go diff --git a/internal/broker/derived.go b/internal/broker/derived.go index e1da042..861b12c 100644 --- a/internal/broker/derived.go +++ b/internal/broker/derived.go @@ -28,6 +28,13 @@ type Consumer struct { // Queue is the queue group, set for a seat's worker so that "exactly one holder" survives a // seat later being relaxed to several. Authority and delivery are kept separate on purpose. Queue string + // Push asks the server to deliver to a subject rather than wait to be pulled. + // + // For the mesh's own consumer, where the controller wants every message to arrive in the one + // loop it already runs: pulling would mean a second goroutine fetching batches and handing + // them over, and a loop that acts on one message at a time is the property the store window + // depends on. A queue group implies this, because a group has nothing to pull from. + Push bool // AckWaitSeconds before an unacknowledged delivery is redelivered. AckWaitSeconds int // MaxDeliver before the message is dead-lettered; zero for the mesh's default. diff --git a/internal/broker/jetstream.go b/internal/broker/jetstream.go index 59f9caf..1867e21 100644 --- a/internal/broker/jetstream.go +++ b/internal/broker/jetstream.go @@ -39,6 +39,14 @@ func Dial(url string, opts ...nats.Option) (*JetStream, error) { return &JetStream{conn: conn, js: js}, nil } +// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat, +// a tool call — where a lost message is answered by the next one or by a timeout the caller +// already handles (design 25 §3). +func (j *JetStream) Conn() *nats.Conn { return j.conn } + +// Context is the JetStream handle, for subscribing to what the consumers above define. +func (j *JetStream) Context() nats.JetStreamContext { return j.js } + func (j *JetStream) Close() { if j.conn != nil { j.conn.Close() @@ -112,7 +120,7 @@ func (j *JetStream) EnsureConsumer(c Consumer) error { // A queue group needs a delivery subject: a pull consumer has no group, and declaring one // without the other is refused by the server with a message that does not say which half is // missing. - if c.Queue != "" { + if c.Queue != "" || c.Push { want.DeliverSubject = "_DELIVER." + c.Name } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 80089fd..1dbb11f 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -143,6 +143,16 @@ func PermissionsFor(p Principal) (Permissions, error) { pub = []string{"mesh.control.>", "mesh.node.>", "mesh.build.>", "$JS.API.>"} sub = []string{"mesh.control.>", "mesh.build.>", "$JS.API.>"} + // The two events it reacts to, and its ack subject on the stream they arrive from + // (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the + // controller a subscriber to every event in the mesh, and its permission list would stop + // saying what it is for. The ack grant below is scoped per stream because the controller's + // consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a + // delivery from EVENTS — a consumer that cannot ack has every message redelivered for + // ever, refused by the list it already has. + sub = append(sub, ControllerFollows...) + pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>") + case KindPerson: // Tools, and nothing else. Every subject a person may publish is a tool call; a person // who could publish an event would be able to claim a module said something. diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 90302ff..7b0d11a 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -150,3 +150,79 @@ func Overlaps() []string { sort.Strings(clashes) return clashes } + +// The mesh's own consumers. +// +// A seat's streams and a module's consumers are derived from declarations (derived.go). These two +// are not: **the controller is not a module and files no manifest**, so its authority and its +// subscriptions cannot come from a declaration that does not exist. They are named here, where the +// mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would +// hear every event in the mesh, which it has no business doing and which would make its permission +// list stop explaining anything. + +// ControllerName is the controller's durable consumer on each stream it reads, and the name its +// ack subject is derived from (nats.go: `$JS.ACK..controller.>`). +const ControllerName = "controller" + +// ControllerFollows are the events the controller reacts to: the catalogue saying a module's +// current version moved, and a catalogue that has just started saying it may have missed builds. +// +// **These carry the local names the manifests hold today**, which still spell an event the way a +// routing key on the bus the mesh has does — `module..` rather than design 29's bare +// verb — so the derived subject names the module twice. It is consistent, and it is what the +// catalogue actually publishes, so it is what the controller must listen to. It changes when those +// names are converted, and not before: a subscription written against the name design 29 specifies +// would be a controller listening to a subject nothing publishes. +var ControllerFollows = []string{ + "mesh.mod.mesh-catalog.event.module.mesh-catalog.upgraded", + "mesh.mod.mesh-catalog.event.module.mesh-catalog.catching-up", +} + +// MeshConsumers is what the controller consumes, in the order a person reads it. +// +// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's, +// not the server's (window.go): a message is held with a nak-and-delay until the controller either +// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being +// held through a store restart — the exact message the stream exists to protect — some minutes +// before the controller had finished deciding about it. +func MeshConsumers() []Consumer { + return []Consumer{ + { + Name: ControllerName, + Stream: "CONTROL", + Push: true, + AckWaitSeconds: 30, + Why: "the controller is the single consumer of what nodes say; explicit ack and no " + + "max-deliver, because the store window's bound is the controller's own", + }, + { + Name: ControllerName, + Stream: "EVENTS", + Filters: ControllerFollows, + Push: true, + AckWaitSeconds: 30, + MaxDeliver: 5, + Why: "the two events the mesh's own controller reacts to; after max-deliver it " + + "dead-letters, because an announcement it cannot act on will not become actionable", + }, + } +} + +// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason +// Asserter is. +type Ensurer interface { + EnsureConsumer(c Consumer) error +} + +// AssertMeshConsumers brings the controller's own consumers into being, and says which one failed. +// +// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the +// refusal names the stream rather than the order. +func AssertMeshConsumers(e Ensurer) error { + for _, c := range MeshConsumers() { + if err := e.EnsureConsumer(c); err != nil { + return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err) + } + } + return nil +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index c879021..edfd456 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -20,8 +20,8 @@ accounts { MESH { users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.API.>", "mesh.build.>", "mesh.control.>", "mesh.node.>"] } - subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.build.>", "mesh.control.>"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "mesh.build.>", "mesh.control.>", "mesh.node.>"] } + subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.build.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.module.mesh-catalog.catching-up", "mesh.mod.mesh-catalog.event.module.mesh-catalog.upgraded"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "enrolment", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { diff --git a/internal/link/bus.go b/internal/link/bus.go index d35b864..b96d3cf 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -92,6 +92,34 @@ type OverNATS struct { JS nats.JetStreamContext } +// The subjects a node publishes on, and the controller listens to. +// +// One tree, and each name says who it is about: `mesh.control..…` is a node's own, which is +// what lets a node's account be granted exactly its own prefix and nothing of any other node's +// (design 25 §2, §4). The two that belong to no node — an enrolment, because a machine enrolling +// has no name the mesh has agreed to yet, and a build's outcome, because a builder is not +// reporting about itself — are named directly. +const ( + // EnrolSubject is where a joining machine asks. Its enrolment user may publish here and + // nowhere else, so a leaked token buys nothing but the chance to enrol. + EnrolSubject = "mesh.control.enrol" + + // BuiltSubject is where a build's outcome lands, for results nobody was waiting for. + BuiltSubject = "mesh.control.built" + + // AliveSubjects is every node's heartbeat. Core NATS, never a stream: a lost heartbeat is the + // next heartbeat, and a stream of them is the mesh's least valuable message competing for + // retention with its most valuable (design 25 §3). + AliveSubjects = "mesh.control.*.alive" +) + +// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the +// message the store-window guarantee is about (ADR 0083). +func ReportSubject(node string) string { return "mesh.control." + node + ".report" } + +// AliveSubject is one node's heartbeat. +func AliveSubject(node string) string { return "mesh.control." + node + ".alive" } + // EventSubject is where a module's event lands. Derived from the emitter, never taken from the // caller: a source that could differ from the subject is an envelope that can lie about its // origin, and on NATS the account's permissions make the subject the authority (design 29 §2). diff --git a/internal/link/protocol.go b/internal/link/protocol.go index d814d15..df7276e 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -80,6 +80,20 @@ type EnrolRequest struct { // it. Nil from a node that found none, which is every converged one. Tunnel *Tunnel `json:"tunnel,omitempty"` + // ReplyTo is where the answer goes, as a field of the request rather than the transport's own + // reply address. + // + // **Because a stream eats the transport's field** (design 25 §2, verified against a running + // server): a message a JetStream consumer delivers has had its reply field claimed for that + // consumer's own ack address, so by the time the controller sees an enrolment, the field names + // where the *controller* must acknowledge, not where the node is waiting. An enrolment is the + // case that matters — a caller waiting on an ephemeral inbox, over a subject the store window + // may legitimately delay by several nak cycles. + // + // Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the + // field means what it has always meant. + ReplyTo string `json:"reply_to,omitempty"` + // Redelivered is set by the control plane, never sent: the broker handed this request over a // second time. Such a request does not finish an enrolment already spent — the first time may // have answered, and the node holds what it was told. diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go new file mode 100644 index 0000000..3acac49 --- /dev/null +++ b/internal/link/receive_nats.go @@ -0,0 +1,285 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The consume side on the bus being built. +// +// The shape is the AMQP one's, because the seam made them comparable: one loop, one message at a +// time, and the same window deciding. What differs is where a held message lives — and that is the +// whole point of the move. On the bus the mesh has, holding one means keeping an unacknowledged +// delivery in this process, bounded by the prefetch and lost if the controller stops. Here it is a +// `nak` with a delay: the message stays the server's, the controller keeps nothing but the moment +// it first could not take it, and a controller that restarts mid-window has nothing to lose. + +// natsInbound consumes what nodes and modules say over NATS. +type natsInbound struct { + js *broker.JetStream + // follows is the kinds asked for beyond what nodes say (Also). The events those are are the + // only ones the controller subscribes, and only when something is listening. + follows map[string]bool + // since is when the controller first could not take a message, by that message's place in its + // stream. + // + // **A timestamp, not a message.** This is the whole difference the move buys: the AMQP side + // keeps the delivery, and this keeps eight bytes saying when the window opened. A controller + // that restarts loses these and starts the window again, which is correct — it is holding + // nothing, and the messages are all still on the server. + since map[uint64]time.Time +} + +// Nats is the consume side of the bus being built. +func Nats(js *broker.JetStream) Inbound { + return &natsInbound{js: js, follows: map[string]bool{}, since: map[uint64]time.Time{}} +} + +// Also records one more kind to subscribe. Nothing is subscribed here: the controller's consumer on +// the events stream carries both of these as filters, so it is created once, in Receive, with +// whatever was asked for — and not at all when nothing was. +func (n *natsInbound) Also(kind string) error { + switch kind { + case KindModuleMoved, KindCatchUp: + n.follows[kind] = true + return nil + default: + return fmt.Errorf("nothing subscribes %s separately on this bus", kind) + } +} + +func (n *natsInbound) Close() {} + +// Receive consumes until the context ends. +// +// Three subscriptions, and each is a channel the one loop selects on. **Channels rather than +// callbacks**: the library would run a handler on its own goroutine, and the window's bookkeeping — +// which message is held, and since when — is read and written without a lock because the AMQP loop +// never had two. A second goroutine would make that wrong in a way no test would catch. +func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Control)) error { + if err := broker.AssertMeshConsumers(n.js); err != nil { + return err + } + js, conn := n.js.Context(), n.js.Conn() + + // What nodes say, off the CONTROL stream. Bound to the durable the controller asserted rather + // than creating one here: the consumer is an object with a configuration — ack policy, ack + // wait, redelivery — and a client that creates its own would be a second opinion about it. + control := make(chan *nats.Msg, Prefetch) + said, err := js.ChanSubscribe("", control, nats.Bind("CONTROL", broker.ControllerName)) + if err != nil { + return fmt.Errorf("subscribing to what nodes say: %w", err) + } + defer func() { _ = said.Unsubscribe() }() + + // Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because + // they are their own guarantee: a lost one is the next one. + beats := make(chan *nats.Msg, Prefetch) + alive, err := conn.ChanSubscribe(AliveSubjects, beats) + if err != nil { + return fmt.Errorf("subscribing to heartbeats: %w", err) + } + defer func() { _ = alive.Unsubscribe() }() + + // The events the controller follows, when something is listening for them. + var events chan *nats.Msg + if len(n.follows) > 0 { + events = make(chan *nats.Msg, Prefetch) + followed, err := js.ChanSubscribe("", events, nats.Bind("EVENTS", broker.ControllerName)) + if err != nil { + return fmt.Errorf("subscribing to what the catalogue says: %w", err) + } + defer func() { _ = followed.Unsubscribe() }() + } + + // A connection that dropped is said, not discovered. A controller whose bus connection is gone + // is a mesh where nothing can be told anything. + gone := make(chan error, 1) + conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { + select { + case gone <- err: + default: + } + }) + + for { + select { + case <-ctx.Done(): + return nil + case err := <-gone: + return fmt.Errorf("the bus connection dropped: %w", err) + case msg := <-beats: + n.deliver(ctx, act, msg, false) + case msg := <-events: + n.deliver(ctx, act, msg, true) + case msg, ok := <-control: + if !ok { + return errors.New("the bus stopped delivering") + } + n.deliver(ctx, act, msg, true) + } + } +} + +// deliver names one message and hands it to the loop, or drops it where the mesh has no name for +// its subject — which cannot happen through a filter the controller wrote, and is said rather than +// ignored for exactly that reason. +func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Control), + msg *nats.Msg, streamed bool) { + + kind, known := kindOfSubject(msg.Subject) + if !known { + if streamed { + _ = msg.Term() + } + return + } + m := &natsControl{kind: kind, msg: msg, on: n} + if streamed { + // A message with no metadata is not from a stream, whatever it was delivered on, and the + // window has nothing to hold it by. Said by leaving the sequence at zero. + if meta, err := msg.Metadata(); err == nil { + m.seq = meta.Sequence.Stream + m.delivered = meta.NumDelivered + } + } + act(ctx, m) +} + +// kindOfSubject is how this transport's addressing becomes what the mesh calls a message. +// +// By subject, which is the only thing the server enforces: a body claiming to be a report does not +// make it one, and on this bus the subject an account may publish *is* its authority (design 29 +// §2). The mirror of the routing-key table on the bus the mesh has. +func kindOfSubject(subject string) (string, bool) { + switch subject { + case EnrolSubject: + return KindEnrolment, true + case BuiltSubject: + return KindBuilt, true + } + if node, rest, ok := strings.Cut(strings.TrimPrefix(subject, "mesh.control."), "."); ok && + node != "" && !strings.Contains(node, ".") { + switch rest { + case "report": + return KindReport, true + case "alive": + return KindHeartbeat, true + } + } + switch subject { + case broker.ControllerFollows[0]: + return KindModuleMoved, true + case broker.ControllerFollows[1]: + return KindCatchUp, true + } + return "", false +} + +// natsControl is one message from the bus being built, as the controller reads it. +type natsControl struct { + kind string + msg *nats.Msg + on *natsInbound + // seq is this message's place in its stream; zero for a core message, which has none and + // cannot be held. + seq uint64 + // delivered is how many times the server has handed this message over, this time included. + delivered uint64 +} + +func (m *natsControl) Kind() string { return m.kind } +func (m *natsControl) Body() []byte { return m.msg.Data } + +// Redelivered is what the server counted, not what the controller remembers. Which is the answer to +// a question the AMQP side could only guess at across a restart: an enrolment redelivered because +// the controller stopped mid-answer reads as redelivered to the controller that comes back. +func (m *natsControl) Redelivered() bool { return m.delivered > 1 } + +func (m *natsControl) HeldFor() time.Duration { + if m.seq == 0 { + return 0 + } + first, held := m.on.since[m.seq] + if !held { + return 0 + } + return time.Since(first) +} + +// About is nothing here, and that is the point. +// +// Setting a held message aside when a newer one about the same thing arrives is what a controller +// holding deliveries in memory can do. A naked message belongs to the server and comes back +// whatever happened meanwhile, so the question "is this the past?" is answered by what the message +// says instead — the digest of the declaration a report is about (window.go, design 25 §3). +func (m *natsControl) About(string) {} + +// Answer publishes to the reply subject the request carries **in its payload**. +// +// Not `Respond`, and not the message's reply field: a message a JetStream consumer delivers has had +// that field claimed for the consumer's own ack address, so answering it would send the reply to +// `$JS.ACK.CONTROL.controller.…` and the enrolling node would wait out its timeout. Verified +// against a running server (design 25 §2), which is why it is a field of the request and this reads +// it from there. +func (m *natsControl) Answer(ctx context.Context, body []byte) error { + var addressed replyAddressed + if err := json.Unmarshal(m.msg.Data, &addressed); err != nil { + return fmt.Errorf("that request cannot be read, so its reply address cannot be: %w", err) + } + if addressed.ReplyTo == "" { + return errors.New("that request named no reply subject in its payload, so nothing can be " + + "told the answer") + } + return m.on.js.Conn().PublishMsg(&nats.Msg{Subject: addressed.ReplyTo, Data: body}) +} + +func (m *natsControl) Took() error { + m.forget() + if m.seq == 0 { + // Core NATS: nothing is keeping it, so there is nothing to settle. + return nil + } + return m.msg.Ack(nats.Context(context.Background())) +} + +// Drop terminates the delivery: understood, and the server is told not to send it again. Different +// from an ack only in the server's own accounting, which is where somebody asking "what happened to +// that message" will look. +func (m *natsControl) Drop() error { + m.forget() + if m.seq == 0 { + return nil + } + return m.msg.Term() +} + +// Hold hands the message back with a delay, and remembers when the window opened. +func (m *natsControl) Hold(after time.Duration) error { + if m.seq == 0 { + return errors.New("a message that is not in a stream cannot be held: nothing is keeping it") + } + if _, already := m.on.since[m.seq]; !already { + m.on.since[m.seq] = time.Now() + } + return m.msg.NakWithDelay(after) +} + +func (m *natsControl) forget() { + if m.on != nil && m.seq != 0 { + delete(m.on.since, m.seq) + } +} + +// replyAddressed is the one field every message that expects an answer carries. +type replyAddressed struct { + ReplyTo string `json:"reply_to,omitempty"` +} diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go new file mode 100644 index 0000000..2f4a58e --- /dev/null +++ b/internal/link/receive_nats_test.go @@ -0,0 +1,365 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The consume side against a real server, because what is being checked is what the server does. +// +// Reasoning cannot answer any of these: whether a nak-with-delay really comes back, whether the +// delay is honoured, whether terminating a delivery really stops it, or whether an answer published +// to an address carried in the payload reaches a caller waiting on its own inbox. Each is a claim +// about a server, so each is asked of one: +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestNats + +func aBus(t *testing.T) *broker.JetStream { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + + // The mesh's own streams and consumers, asserted the way the controller asserts them — and + // torn down after, so one test's held message is never another's surprise. + for _, s := range broker.MeshStreams() { + _ = js.Context().DeleteStream(s.Name) + } + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + for _, s := range broker.MeshStreams() { + _ = js.Context().DeleteStream(s.Name) + } + }) + return js +} + +// serving1 is a controller reading from a real bus, and a way to stop it. +func servingOn(t *testing.T, js *broker.JetStream, l Listener) (*Server, func()) { + t.Helper() + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + listener: l, log: quiet()} + ctx, stop := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = s.Serve(ctx) }() + return s, func() { + stop() + <-done + } +} + +// counted records reports and can be told to refuse them, from another goroutine. +type counted struct { + mu sync.Mutex + err error + heard []Report +} + +func (c *counted) Heard(_ context.Context, r Report) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.err != nil { + return c.err + } + c.heard = append(c.heard, r) + return nil +} + +func (c *counted) refusing(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.err = err +} + +func (c *counted) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.heard) +} + +func eventually(t *testing.T, what string, is func() bool) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + if is() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s did not happen within the wait", what) +} + +// A report published by a node reaches the controller, is recorded, and is acknowledged — so the +// stream does not hold it. A work queue is the check: what is acknowledged leaves it. +func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "a report being recorded", func() bool { return store.count() == 1 }) + eventually(t, "the report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +// **The store window, in the server.** A report the store cannot take is naked with a delay and +// comes back; once the store is there it is recorded and leaves the stream. The controller holds +// nothing in the meantime — which is what the sequence check below is for: the message is still on +// the server while it waits. +func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up"))) + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + + // Held: the message is the server's, unacknowledged, and still in the stream. + eventually(t, "the report being redelivered at least once", func() bool { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + return err == nil && info.NumRedelivered >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 1 { + t.Fatalf("a held report did not stay on the server: %+v, %v", info, err) + } + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that was refusing it") + } + + store.refusing(nil) + eventually(t, "the report being recorded once the store was back", + func() bool { return store.count() == 1 }) + eventually(t, "the recorded report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +// A report about a declaration the mesh has moved past is settled without being acted on, and +// leaves the stream rather than coming back for ever. +func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { + js := aBus(t) + store := &sentAndHeardSafely{sent: "d2"} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the superseded report leaving the stream", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report about a superseded declaration was acted on") + } +} + +// sentAndHeardSafely is sentAndHeard, read from two goroutines. +type sentAndHeardSafely struct { + mu sync.Mutex + sent string + heard []Report +} + +func (s *sentAndHeardSafely) Heard(_ context.Context, r Report) error { + s.mu.Lock() + defer s.mu.Unlock() + s.heard = append(s.heard, r) + return nil +} + +func (s *sentAndHeardSafely) Outstanding(context.Context, string) (string, error) { + return s.sent, nil +} + +func (s *sentAndHeardSafely) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.heard) +} + +// **An enrolment answered through a reply address the stream would have eaten.** +// +// The caller waits on its own inbox and states that address in the request's payload. The check is +// that the answer arrives there — which is the whole reason the address is a field rather than the +// transport's reply, and this is the test design 25 §2 asks for so the reason cannot quietly become +// folklore. +func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { + js := aBus(t) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + inbox := nats.NewInbox() + answers, err := js.Conn().SubscribeSync(inbox) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox}) + if _, err := js.Context().Publish(EnrolSubject, body); err != nil { + t.Fatal(err) + } + + msg, err := answers.NextMsg(8 * time.Second) + if err != nil { + t.Fatalf("no answer reached the address the request named: %v", err) + } + var reply EnrolReply + if err := json.Unmarshal(msg.Data, &reply); err != nil { + t.Fatal(err) + } + if !reply.Accepted || reply.Node != "anchor" { + t.Fatalf("the answer was not the mesh's: %+v", reply) + } + // And the address really is not the one the transport carried: what the consumer saw was its + // own ack subject, which is why this had to travel in the payload. + if msg.Subject != inbox { + t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject) + } +} + +// enrolsAs answers every request the same way. +type enrolsAs struct{ reply EnrolReply } + +func (e enrolsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) { return e.reply, nil } + +// A heartbeat is core NATS: it reaches the controller and nothing is persisted, so the stream the +// reports live in stays empty. +func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + // Given time to subscribe: a core subscription that is not yet up misses what is published, + // which is the guarantee a heartbeat has and not a fault. + eventually(t, "the heartbeat subscription coming up", func() bool { + body, _ := json.Marshal(Alive{Node: "anchor"}) + _ = js.Conn().Publish(AliveSubject("anchor"), body) + _ = js.Conn().Flush() + return store.count() >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 0 { + t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+ + "for retention with its most valuable: %+v, %v", info, err) + } +} + +// The two events the controller follows arrive over one durable consumer with two filters, and it +// can acknowledge them. +// +// **Both halves are the point.** A consumer with several filter subjects is a 2.10 feature and this +// is the first thing in the mesh to use one; and a delivery from the events stream is acknowledged +// on a different ack subject from a delivery from the control stream, which the controller's own +// permission list has to cover or every announcement is redelivered for ever. +func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) { + js := aBus(t) + told := &toldAbout{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(told); err != nil { + t.Fatal(err) + } + if err := s.Answers(replaysWith{}); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil { + t.Fatal(err) + } + + eventually(t, "the catalogue's upgrade reaching the controller", + func() bool { return told.count() == 1 }) + eventually(t, "both announcements being acknowledged", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 + }) +} + +type toldAbout struct { + mu sync.Mutex + saw []Upgraded + fail error +} + +func (u *toldAbout) Upgraded(_ context.Context, m Upgraded) error { + u.mu.Lock() + defer u.mu.Unlock() + if u.fail != nil { + return u.fail + } + u.saw = append(u.saw, m) + return nil +} + +func (u *toldAbout) count() int { + u.mu.Lock() + defer u.mu.Unlock() + return len(u.saw) +} + +// A store that never comes back: the report is let go once the bound passes, and it leaves the +// stream rather than being held for ever. The bound is the controller's, not the server's — nothing +// here sets max-deliver, and that is deliberate (streams.go). +func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused"))) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + listener: store, log: quiet(), giveUp: 1500 * time.Millisecond} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the report being let go once the bound passed", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that never came back") + } +} -- 2.54.0 From e65b3950cc9c2060df6c276a1fb8d8e86ee87c7c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 01:24:46 +0200 Subject: [PATCH 22/39] A node's declaration consumer, which only the mesh can make MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host's account reaches no part of the JetStream API — correctly, because the controller is the only writer of consumer definitions — so the object a node reads its declarations through has to be waiting before the host binds to it, and nothing created one. Named after the node, because the node's own ack grant is `$JS.ACK.NODES..>` and a consumer named anything else is one the host cannot acknowledge a delivery from. No max-deliver, and a five-minute ack wait: a declaration is settled only after the node has applied it and reported, which is minutes on a machine pulling images, and the stream holds exactly one message per node — so there is nothing to dead-letter, only one message to redeliver for as long as that node is away. Asserted on start as well as created at enrolment, for the reason the streams are: a mesh raised from a restored backup has node records and no consumers, and a node whose consumer is missing hears nothing while everything else about it looks correct. The test sets the consumer against the grant the node actually gets, because each of the three ways of getting it wrong is silent: a wrong name cannot ack, a wrong filter reads another node's declarations, and a pull consumer is one a host has no authority to bind. --- internal/broker/derived.go | 39 +++++++++++++++++++++++++++++++++ internal/broker/derived_test.go | 36 ++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/internal/broker/derived.go b/internal/broker/derived.go index 861b12c..97d3174 100644 --- a/internal/broker/derived.go +++ b/internal/broker/derived.go @@ -146,6 +146,45 @@ func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) }, true } +// NodeConsumer is the durable consumer a node reads its own declaration through. +// +// **Derived from a node existing, and created by the controller, because a host cannot create it.** +// A host's account may subscribe its own declaration subject and publish its own ack subject, and +// reaches no part of the JetStream API — which is correct (the controller is the only writer of +// consumer definitions, design 25 §3) and means the consumer must be waiting before the host binds +// to it. Named after the node, because the node's ack grant is `$JS.ACK.NODES..>` and a +// consumer named anything else is one the host cannot acknowledge a delivery from. +// +// **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied +// it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot +// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node +// anyway — so there is exactly one message per node to redeliver, for as long as that node is away. +func NodeConsumer(node string) Consumer { + return Consumer{ + Name: node, + Stream: "NODES", + Filters: []string{"mesh.node." + node + ".declare"}, + Push: true, + AckWaitSeconds: 300, + Why: "how " + node + " hears what it should be; last-per-subject, so a node that was away " + + "gets exactly the current declaration and nothing older", + } +} + +// AssertNodeConsumers brings every known node's declaration consumer into being. +// +// Asserted on start as well as created at enrolment, for the reason the streams are: a mesh raised +// from a restored backup, or one whose bus was recreated, has node records and no consumers, and a +// node whose consumer is missing hears nothing while everything else about it looks correct. +func AssertNodeConsumers(e Ensurer, nodes []string) error { + for _, n := range nodes { + if err := e.EnsureConsumer(NodeConsumer(n)); err != nil { + return fmt.Errorf("asserting how %s hears its declaration: %w", n, err) + } + } + return nil +} + // AllOverlaps reports subject filters claimed by more than one stream, across the mesh's own and // every derived one. // diff --git a/internal/broker/derived_test.go b/internal/broker/derived_test.go index bafe7d0..b4529d4 100644 --- a/internal/broker/derived_test.go +++ b/internal/broker/derived_test.go @@ -117,3 +117,39 @@ func TestASeatsStreamIsNamedAfterTheSeat(t *testing.T) { t.Fatalf("unexpected stream name %q", name) } } + +// A node hears its declaration through a consumer only the controller can make. +// +// The three things that would each break it silently: a name other than the node's is one the host +// cannot acknowledge a delivery from, because its ack grant is derived from the node's name; a +// filter other than its own declaration subject is a node reading another's; and a pull consumer is +// one the host cannot bind a channel to without creating something, which it has no authority for. +func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) { + c := NodeConsumer("anchor") + if c.Name != "anchor" { + t.Fatalf("named %q, so the node cannot ack from it: its grant is $JS.ACK.NODES.anchor.>", c.Name) + } + if c.Stream != "NODES" { + t.Fatalf("on stream %q rather than the one declarations live in", c.Stream) + } + if len(c.Filters) != 1 || c.Filters[0] != "mesh.node.anchor.declare" { + t.Fatalf("filters %v, which is not this node's own declaration and nothing else", c.Filters) + } + if !c.Push { + t.Fatal("pulled, which a host cannot do: pulling needs the JetStream API and a host reaches none of it") + } + if c.MaxDeliver != 0 { + t.Fatalf("max-deliver %d: a declaration a node has not taken yet is not one to dead-letter, "+ + "because the stream holds exactly one per node", c.MaxDeliver) + } + + // And the grant the node actually gets has to match, or none of the above matters. + perms, err := PermissionsFor(Principal{Kind: KindNode, Node: "anchor"}) + if err != nil { + t.Fatal(err) + } + // Without the ack grant every declaration a node receives is redelivered for ever; without the + // subscribe grant its consumer delivers to nobody. + has(t, perms.Publish, "$JS.ACK.NODES."+c.Name+".>") + has(t, perms.Subscribe, c.Filters[0]) +} -- 2.54.0 From 7180a273a283a11958a7dd9679e4b26a8e38adb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 01:31:34 +0200 Subject: [PATCH 23/39] The enrolment user is per token, and it has an inbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 25 §6 says an enrolling node subscribes the inbox its own token derives. It had none: `sub` was empty, so a node would publish its request and wait out its timeout against a mesh that had answered — the handshake could not have completed. And there was one shared `enrolment` user, which cannot carry that inbox at all: a permission belongs to a user, so an inbox per token means a user per token. Named after the node, which **is** the token's id — a token is issued for a node record, the mesh holds one live claim per record, and the node's name is the one identifier both sides have before anything else is agreed. It is also exactly what the other transport does, where the account is named after the node and the secret is its password. A nameless enrolment user is now refused rather than composed into `_INBOX.enrol..>`: an empty subject token, and worse, one every nameless enrolment user would share — which is one machine able to read the credentials sealed to another. Still to wire: something that composes one of these per live token. Nothing composes enrolment users yet, on either bus — on the old one the account is made imperatively through the broker's management API when a token is issued, and here there is no management API, so issuing a token has to recompose the server's configuration. That is the remaining half of enrolment on the new bus. --- internal/broker/nats.go | 29 ++++++++++++++++++++++++-- internal/broker/nats_golden_test.go | 2 +- internal/broker/nats_test.go | 21 ++++++++++++++++--- internal/broker/testdata/composed.conf | 4 ++-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 1dbb11f..83c0e87 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -13,6 +13,7 @@ package broker import ( + "errors" "fmt" "regexp" "sort" @@ -94,7 +95,16 @@ func (p Principal) Username() string { case KindController: return "controller" case KindEnrolment: - return "enrolment" + // Per token, not one shared user. **The inbox is the reason**: with a single `enrolment` + // user every machine enrolling at once could read every other's answer, and an answer + // carries that node's credentials sealed to it. Design 25 §6 says the inbox a token + // derives, and a permission belongs to a user, so the user is per token. + // + // Named after the node, which **is** the token's id: a token is issued for a node record, + // the mesh holds one live claim per record, and the node's name is the one identifier both + // sides already have before anything else is agreed. It is also exactly what the other + // transport does, where the account is named after the node and the secret is its password. + return "enrol." + p.Node } return "" } @@ -172,8 +182,23 @@ func PermissionsFor(p Principal) (Permissions, error) { case KindEnrolment: // A leaked token is useless for anything but enrolling: it cannot read a declaration, hear // an event, or subscribe any inbox but the one its own token derives (design 25 §6). + // + // **The inbox was missing and the handshake could not have completed without it.** An + // enrolling node publishes its request and waits on an address it states in the payload; + // with nothing to subscribe it waits out its timeout against a mesh that answered. Its own + // and no wider: `_INBOX.enrol..>`, so what is sealed to one machine cannot be read by + // another enrolling beside it. + if p.Node == "" { + // Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty + // token in it — and worse, one every nameless enrolment user would share. A shared + // enrolment inbox is one machine able to read the credentials sealed to another. + return Permissions{}, errors.New( + "an enrolment user names no node, so its inbox would be shared with every other " + + "enrolment: a token is issued for a node record, and that record's name is " + + "the token's id") + } pub = []string{"mesh.control.enrol"} - sub = []string{} + sub = []string{p.inbox()} case KindNode: // A host publishes its own node's control traffic and subscribes its own declaration — diff --git a/internal/broker/nats_golden_test.go b/internal/broker/nats_golden_test.go index 40f2131..6c3b2da 100644 --- a/internal/broker/nats_golden_test.go +++ b/internal/broker/nats_golden_test.go @@ -20,7 +20,7 @@ func TestTheComposedConfigMatchesTheGolden(t *testing.T) { TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, []Principal{ {Kind: KindController, PasswordHash: "$2a$11$cccccccccccccccccccccc"}, - {Kind: KindEnrolment, PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"}, + {Kind: KindEnrolment, Node: "one", PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"}, {Kind: KindNode, Node: "one", PasswordHash: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn"}, {Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat}, Serves: []string{"status"}, PasswordHash: "$2a$11$tttttttttttttttttttttt"}, diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index f1aa154..f5c2500 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -115,12 +115,27 @@ func TestAHostIsConfinedToItsOwnNode(t *testing.T) { // A leaked enrolment token is useless for anything but enrolling (design 25 §6). func TestTheEnrolmentUserCanOnlyEnrol(t *testing.T) { - perms, _ := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}) + perms, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.control.enrol" { t.Fatalf("enrolment may publish %v", perms.Publish) } - if len(perms.Subscribe) != 0 { - t.Fatalf("enrolment may subscribe %v, and should hear nothing", perms.Subscribe) + // Its own inbox and nothing else. **Nothing else** is the point: no declaration, no event, and + // no other machine's answer — and the inbox itself is needed, because a node that cannot + // subscribe one waits out its timeout against a mesh that answered. + if len(perms.Subscribe) != 1 || perms.Subscribe[0] != "_INBOX.enrol.anchor.>" { + t.Fatalf("enrolment may subscribe %v, which is not its own inbox alone", perms.Subscribe) + } +} + +// An enrolment user that names no node is refused: its inbox would be an empty subject token, and +// one that every nameless enrolment user shared — which is one machine reading the credentials +// sealed to another. +func TestAnEnrolmentUserWithoutANodeIsRefused(t *testing.T) { + if _, err := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}); err == nil { + t.Fatal("an enrolment user with no node was composed, so its inbox is shared") } } diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index edfd456..489435a 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,9 +24,9 @@ accounts { subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.build.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.module.mesh-catalog.catching-up", "mesh.mod.mesh-catalog.event.module.mesh-catalog.upgraded"] } allow_responses: { max: 1, ttl: "1m" } } } - { user: "enrolment", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { + { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { publish: { allow: ["mesh.control.enrol"] } - subscribe: { allow: [] } + subscribe: { allow: ["_INBOX.enrol.one.>"] } } } { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: { publish: { allow: ["$JS.ACK.NODES.one.>", "mesh.control.one.>"] } -- 2.54.0 From 0560c792d84d1d25c8bcff2d672e89c7b789a07f Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 01:44:53 +0200 Subject: [PATCH 24/39] 1.7, first half: the mesh can say who its bus users are, and hold their keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pieces the composer has been waiting for since it was written. **The credential has to outlive its own minting.** On the bus the mesh runs on today an account is a management call: mint a password, hand it over, seal the plaintext to whoever will use it, keep nothing — which works because the broker remembers. Here the users are one file, rewritten whenever any of it changes, so keeping nothing would mean the first person's access change silently blanking every module's password. So a bus user's bcrypt hash is now recorded, keyed by the username the file needs, and the plaintext comes back exactly once. Verified against a real store that the hash verifies the password it was made from, that the password itself is not in there, that minting again rotates rather than adds, and that forgetting a node takes its host's and its modules' credentials with it. **Permissions are not stored, and that is the point.** Only the credential is kept. Authority is derived from what each module declares, every time the file is written (ADR 0043) — a stored permission list would be a second account of a user's authority, able to disagree with the records it came from, and both would look internally consistent while they did. `Users` derives the list: the controller always first and always present, one user per node, one per module per node, one per live token, one per person. Two users with one name is refused where both can be named, rather than left to be whichever one the server happened to read. A user the mesh has never minted a password for is *named* rather than dropped or written as a user anybody is: that is an ordinary situation with an obvious remedy, and the caller decides whether a partial file is worth writing. What remains of 1.7: delivering the file to the node that runs the server, and minting at enrolment and assignment — which is transport-coupled, because a node on the old bus must not be handed a credential for the new one. --- internal/broker/users.go | 125 ++++++++++++++ internal/broker/users_test.go | 156 ++++++++++++++++++ internal/inventory/bususers.go | 138 ++++++++++++++++ internal/inventory/bususers_test.go | 123 ++++++++++++++ .../0033-the-bus-keeps-its-users-hashes.sql | 36 ++++ 5 files changed, 578 insertions(+) create mode 100644 internal/broker/users.go create mode 100644 internal/broker/users_test.go create mode 100644 internal/inventory/bususers.go create mode 100644 internal/inventory/bususers_test.go create mode 100644 internal/inventory/migrations/0033-the-bus-keeps-its-users-hashes.sql diff --git a/internal/broker/users.go b/internal/broker/users.go new file mode 100644 index 0000000..f6fc289 --- /dev/null +++ b/internal/broker/users.go @@ -0,0 +1,125 @@ +package broker + +import ( + "fmt" + "sort" +) + +// Every user the composed file should contain, derived from what the mesh knows. +// +// **The list is derived, never kept.** A stored user list would be a second account of who may +// reach the bus, able to disagree with the records it came from — and the disagreement would be +// invisible, because both would look internally consistent. So this is a pure function of the +// mesh's records, run again every time the file is written. +// +// Records are mirrored into this package's own types rather than imported from the catalogue, for +// the reason DeclaredSeat is: composing authority is a different job from parsing a manifest, and +// this package stays free of the other's types so a change to a manifest field cannot quietly widen +// a permission. + +// Declared is one module on one node, as composing its authority needs it. +type Declared struct { + Module string + Emits []string + Consumes []string + Serves []string + // Holds are the seats this module claims, with the protocol each seat declares. A seat the + // mesh defines for itself declares no protocol, so holding one grants nothing on the bus — + // which is right: those seats are about who does a job, not about who may say what. + Holds []Seat + // Uses are the seats this module sends to. + Uses []Seat +} + +// Records is what composing a user list needs to know about the mesh, and nothing more. +type Records struct { + // Nodes is every machine the mesh knows. Each gets a host user. + Nodes []string + // Assigned is the modules on each node, as they declare themselves. + Assigned map[string][]Declared + // Enrolling is every node with a live token — one enrolment user each, because the inbox an + // answer goes to is scoped to the token and a shared one is one machine reading another's + // sealed credentials (design 25 §6). + Enrolling []string + // People is each person's name against the tools they may invoke, `*` for an administrator. + People map[string][]string +} + +// Users is every user the composed file should contain, in the order it will be written. +// +// The controller is always first and always present: a mesh whose own controller is not in the file +// is a mesh that cannot be told anything, and there is no state of the records in which that is +// correct. +func Users(r Records) ([]Principal, error) { + out := []Principal{{Kind: KindController}} + + for _, node := range sortedCopy(r.Nodes) { + out = append(out, Principal{Kind: KindNode, Node: node}) + for _, d := range r.Assigned[node] { + out = append(out, Principal{ + Kind: KindModule, Node: node, Module: d.Module, + Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves, + Holds: d.Holds, Uses: d.Uses, + }) + } + } + for _, node := range sortedCopy(r.Enrolling) { + out = append(out, Principal{Kind: KindEnrolment, Node: node}) + } + for _, person := range sortedNames(r.People) { + out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]}) + } + + // Refused here rather than discovered by the server. Two users with one name is a file the + // server reads as one of them, and which one depends on the order — so a module assigned to a + // node twice, or a person named after nothing, is a composition that must not be written. + seen := map[string]string{} + for _, p := range out { + name := p.Username() + if name == "" || name == "." { + return nil, fmt.Errorf("a %s user has no name, so nothing could authenticate as it", p.Kind) + } + if first, already := seen[name]; already { + return nil, fmt.Errorf( + "two users would be called %q (a %s and a %s): the server would read the file as "+ + "one of them, and which one depends on the order", name, first, p.Kind) + } + seen[name] = string(p.Kind) + } + return out, nil +} + +// WithPasswords fills each user's hash from what the mesh minted, and says which users have none. +// +// **Separated from Users because they fail differently.** A user missing from the records is a bug +// in deriving them; a user with no password is a step that has not happened yet — a module assigned +// but never given a credential, a node enrolled before this existed. The second is ordinary and its +// remedy is to mint one, so it is named rather than returned as an error, and the caller decides +// whether a partial composition is worth writing. +func WithPasswords(principals []Principal, hashes map[string]string) (filled []Principal, missing []string) { + for _, p := range principals { + hash, ok := hashes[p.Username()] + if !ok || hash == "" { + missing = append(missing, p.Username()) + continue + } + p.PasswordHash = hash + filled = append(filled, p) + } + return filled, missing +} + +func sortedCopy(in []string) []string { + out := append([]string(nil), in...) + sort.Strings(out) + return out +} + +func sortedNames(in map[string][]string) []string { + out := make([]string, 0, len(in)) + for k := range in { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/broker/users_test.go b/internal/broker/users_test.go new file mode 100644 index 0000000..5787b9e --- /dev/null +++ b/internal/broker/users_test.go @@ -0,0 +1,156 @@ +package broker + +import ( + "strings" + "testing" +) + +// Deriving the bus's user list from the mesh's records. +// +// Every test here is about a way the list could be wrong that the server would not tell anybody +// about: a user missing, a user named twice, a user with authority it did not declare. + +func someRecords() Records { + return Records{ + Nodes: []string{"two", "one"}, + Assigned: map[string][]Declared{ + "one": {{Module: "telegram", Serves: []string{"status"}}}, + "two": {{Module: "shop", Emits: []string{"order.placed"}}}, + }, + Enrolling: []string{"three"}, + People: map[string][]string{"ada": {"mesh-catalog.catalog_tools"}}, + } +} + +func namesOf(t *testing.T, r Records) []string { + t.Helper() + users, err := Users(r) + if err != nil { + t.Fatal(err) + } + out := make([]string, 0, len(users)) + for _, u := range users { + out = append(out, u.Username()) + } + return out +} + +// The controller is always there. A mesh whose own controller is not in the file is a mesh that +// cannot be told anything, and there is no state of the records in which that is correct. +func TestTheControllerIsAlwaysInTheList(t *testing.T) { + for _, r := range []Records{{}, someRecords()} { + names := namesOf(t, r) + if len(names) == 0 || names[0] != "controller" { + t.Fatalf("the controller is not first in %v", names) + } + } +} + +// One user per node, one per module per node, one per live token and one per person — and nothing +// else, because a user nobody derived is a user nobody can explain. +func TestEveryRecordBecomesExactlyOneUser(t *testing.T) { + names := namesOf(t, someRecords()) + want := []string{ + "controller", + "node.one", "one.telegram", + "node.two", "two.shop", + "enrol.three", + "person.ada", + } + if strings.Join(names, ",") != strings.Join(want, ",") { + t.Fatalf("derived %v\n want %v", names, want) + } +} + +// Two users with one name is a file the server reads as one of them, and which one depends on the +// order. Refused here, where both can be named, rather than left to be whichever the server picked. +func TestTwoUsersWithOneNameAreRefused(t *testing.T) { + r := someRecords() + r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: "telegram"}) + _, err := Users(r) + if err == nil { + t.Fatal("a module assigned twice to one node composed two users with one name") + } + if !strings.Contains(err.Error(), "one.telegram") { + t.Fatalf("the refusal does not name the user: %v", err) + } +} + +// A module's authority is what it declared and nothing more, carried through the derivation intact — +// because this is the step where a mistake would grant something no manifest asked for. +func TestAModulesAuthorityIsWhatItDeclared(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}} + users, err := Users(Records{ + Nodes: []string{"one"}, + Assigned: map[string][]Declared{"one": {{ + Module: "shop", Emits: []string{"order.placed"}, Uses: []Seat{seat}, + }}}, + }) + if err != nil { + t.Fatal(err) + } + perms, err := PermissionsFor(users[len(users)-1]) + if err != nil { + t.Fatal(err) + } + has(t, perms.Publish, "mesh.mod.shop.event.order.placed") + has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send") + // A seat it uses, not one it holds: it may submit work and may not publish the seat's own + // events, or it could lie about outcomes on a role somebody else fills. + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered") + hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send") +} + +// A user the mesh has never minted a password for is named rather than silently dropped or +// composed as a user anybody is. It is an ordinary situation — a module assigned a moment ago — and +// the remedy is to mint one, so the caller decides whether to write a partial file. +func TestAUserWithNoPasswordIsNamedRatherThanWritten(t *testing.T) { + users, err := Users(someRecords()) + if err != nil { + t.Fatal(err) + } + filled, missing := WithPasswords(users, map[string]string{ + "controller": "$2a$hash", "node.one": "$2a$hash", + }) + if len(filled) != 2 { + t.Fatalf("composed %d users from two hashes", len(filled)) + } + if len(missing) != len(users)-2 { + t.Fatalf("%d users are missing a password, of %d: %v", len(missing), len(users), missing) + } + for _, p := range filled { + if p.PasswordHash == "" { + t.Fatalf("%s was kept with no password, which is a user anybody is", p.Username()) + } + } +} + +// And the whole thing composes: records in, a file the server would read out. +func TestRecordsComposeIntoAFile(t *testing.T) { + users, err := Users(someRecords()) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for _, u := range users { + hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22) + } + filled, missing := WithPasswords(users, hashes) + if len(missing) != 0 { + t.Fatalf("users with no password: %v", missing) + } + got, err := Compose(Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data", + TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, filled) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `user: "controller"`, `user: "node.one"`, `user: "one.telegram"`, + `user: "enrol.three"`, `user: "person.ada"`, + `"_INBOX.enrol.three.>"`, `"mesh.mod.mesh-catalog.tool.catalog_tools"`, + } { + if !strings.Contains(got, want) { + t.Errorf("the composed file does not contain %s", want) + } + } +} diff --git a/internal/inventory/bususers.go b/internal/inventory/bususers.go new file mode 100644 index 0000000..c9a43a7 --- /dev/null +++ b/internal/inventory/bususers.go @@ -0,0 +1,138 @@ +package inventory + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "golang.org/x/crypto/bcrypt" +) + +// The bus's own users, as records. +// +// **Only the credential is kept here.** A user's *authority* is derived from what its module +// declares, every time the file is written (novox/hq ADR 0043) — a stored copy of a permission list +// would be a second account of a user's authority, able to disagree with the first, and the +// disagreement would be invisible until somebody compared a composed file with a manifest. +// +// What cannot be derived is the password, and on the bus being built it has to outlive its own +// minting: the whole user list is one file, rewritten whenever any of it changes, so a person's +// access change would blank every module's password if the mesh kept nothing (design 25 §4, and the +// migration beside this). + +// BusUser is one user of the bus, as the mesh records it. +type BusUser struct { + Username string + Kind string + Node string + Module string + // PasswordHash is what the composed file carries. The plaintext is returned once, by Mint, and + // then exists only where it was sealed. + PasswordHash string +} + +// The kinds of bus user the mesh records. The same words the composer uses, so a row and a +// principal do not need a translation table between them. +const ( + BusController = "controller" + BusNode = "node" + BusModule = "module" + BusEnrolment = "enrolment" + BusPerson = "person" +) + +// MintBusPassword makes a bus password and records its hash under a username, replacing whatever was +// there, and returns the plaintext **once**. +// +// **Once is the whole contract.** The caller seals it to whoever will use it — into an enrolment +// reply, into a module's sealed environment — and the mesh keeps only the hash, so a credential is +// never recoverable from the store. A caller that loses it must mint again, which is a rotation and +// is meant to feel like one. +func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) { + if u.Username == "" || u.Kind == "" { + return "", errors.New("a bus user needs a username and a kind") + } + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", fmt.Errorf("cannot generate a bus password: %w", err) + } + password := base64.RawURLEncoding.EncodeToString(raw) + + // The cost the server will pay on every connection. Left at the library's default rather than + // raised: a node reconnecting after a network blip pays it, and the mesh's own links reconnect + // far more often than a person logs in anywhere. + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", fmt.Errorf("cannot hash a bus password: %w", err) + } + + if _, err := i.store.Pool().Exec(ctx, + `insert into bus_user (username, kind, node, module, password_hash) + values ($1, $2, $3, $4, $5) + on conflict (username) do update + set kind = excluded.kind, node = excluded.node, module = excluded.module, + password_hash = excluded.password_hash, minted_at = now()`, + u.Username, u.Kind, u.Node, u.Module, string(hash)); err != nil { + return "", fmt.Errorf("cannot record the bus user %s: %w", u.Username, err) + } + return password, nil +} + +// BusUsers is every user the composed file should contain, by username. +// +// Returned as a map because the composer asks by username: the principals are derived from records +// elsewhere, and this is only what each one's password is. A principal with no row here has no +// password, and the composer refuses it rather than writing a user anybody is. +func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) { + rows, err := i.store.Pool().Query(ctx, + `select username, kind, node, module, password_hash from bus_user order by username`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]BusUser{} + for rows.Next() { + var u BusUser + if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash); err != nil { + return nil, err + } + out[u.Username] = u + } + return out, rows.Err() +} + +// BusUserHash is one user's hash, or false when the mesh has never minted one for it. +func (i *Inventory) BusUserHash(ctx context.Context, username string) (string, bool, error) { + var hash string + err := i.store.Pool().QueryRow(ctx, + `select password_hash from bus_user where username = $1`, username).Scan(&hash) + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + return hash, err == nil, err +} + +// ForgetBusUser removes one user, so the next composition does not contain it. +// +// **Removal is what makes revocation real here.** On a bus with a management call, deleting an +// account ends its connections; here the credential stops working when the file no longer names it, +// which is the next composition — so forgetting the row and composing are one act, and a caller +// that does the first without the second has revoked nothing. +func (i *Inventory) ForgetBusUser(ctx context.Context, username string) error { + _, err := i.store.Pool().Exec(ctx, `delete from bus_user where username = $1`, username) + return err +} + +// ForgetBusUsersOf removes every user belonging to one node — its host's, and every module assigned +// to it. What a forgotten node leaves behind on the bus is otherwise a set of credentials for a +// machine the mesh no longer knows. +func (i *Inventory) ForgetBusUsersOf(ctx context.Context, node string) error { + if node == "" { + return errors.New("forgetting the bus users of no node would forget every user that has none") + } + _, err := i.store.Pool().Exec(ctx, `delete from bus_user where node = $1`, node) + return err +} diff --git a/internal/inventory/bususers_test.go b/internal/inventory/bususers_test.go new file mode 100644 index 0000000..7af4523 --- /dev/null +++ b/internal/inventory/bususers_test.go @@ -0,0 +1,123 @@ +package inventory + +import ( + "context" + "testing" + + "golang.org/x/crypto/bcrypt" +) + +// The bus's users as records — against a real store, because what is being checked is that the +// column exists, the upsert behaves, and a plaintext is returned exactly once. + +func aBusUser(module string) BusUser { + return BusUser{Username: "one." + module, Kind: BusModule, Node: "one", Module: module} +} + +// The plaintext comes back once and the store keeps only a hash that verifies against it. **A +// credential recoverable from the mesh's store is one whose blast radius is the store's**, so what +// is asserted is that the password is not in there. +func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + password, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if password == "" { + t.Fatal("no password came back, so nothing can be sealed to the module") + } + + hash, known, err := inv.BusUserHash(ctx, "one.shop") + if err != nil || !known { + t.Fatalf("the user was not recorded: %v %v", known, err) + } + if hash == password { + t.Fatal("the store holds the password itself") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil { + t.Fatalf("the recorded hash does not verify the password it was made from: %v", err) + } +} + +// Minting again replaces what was there rather than failing or adding a second row: that is a +// rotation, and the old credential stops working at the next composition. +func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + first, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + second, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("minting twice produced the same password") + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if len(users) != 1 { + t.Fatalf("%d users after two mints for one name", len(users)) + } + hash := users["one.shop"].PasswordHash + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil { + t.Fatal("the kept hash is not the newest password's") + } + if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil { + t.Fatal("the previous password still verifies, so a rotation revoked nothing") + } +} + +// Forgetting a node takes every credential that belonged to it — its host's and every module +// assigned to it. What a forgotten node leaves behind otherwise is a working set of credentials for +// a machine the mesh no longer knows. +func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + for _, u := range []BusUser{ + {Username: "node.one", Kind: BusNode, Node: "one"}, + aBusUser("shop"), + {Username: "node.two", Kind: BusNode, Node: "two"}, + {Username: "controller", Kind: BusController}, + } { + if _, err := inv.MintBusPassword(ctx, u); err != nil { + t.Fatal(err) + } + } + if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil { + t.Fatal(err) + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := users["node.one"]; still { + t.Fatal("a forgotten node's host credential still works") + } + if _, still := users["one.shop"]; still { + t.Fatal("a module on a forgotten node still has a credential") + } + // And nothing else went with it: the controller has no node, and another machine's user is + // another machine's. + for _, kept := range []string{"node.two", "controller"} { + if _, ok := users[kept]; !ok { + t.Fatalf("%s was removed with another node's users", kept) + } + } +} + +// Forgetting the users of no node would forget every user that has none — the controller and every +// person — so it is refused rather than run. +func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { + inv := ForTest(t) + if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil { + t.Fatal("forgetting the bus users of no node was allowed") + } +} diff --git a/internal/inventory/migrations/0033-the-bus-keeps-its-users-hashes.sql b/internal/inventory/migrations/0033-the-bus-keeps-its-users-hashes.sql new file mode 100644 index 0000000..0ac7ee0 --- /dev/null +++ b/internal/inventory/migrations/0033-the-bus-keeps-its-users-hashes.sql @@ -0,0 +1,36 @@ +-- Every bus user's password hash, because the file has to be written again. +-- +-- novox/hq design 25 §4, task 1.7. On the bus the mesh runs on today an account is created by a +-- management call: the mesh mints a password, hands it over, seals the plaintext to whoever will +-- use it, and keeps nothing. That works because the broker remembers. +-- +-- The bus being built has no management call — its users are a file the controller composes, and +-- **the whole file is written every time any of it changes**. So the first person's access change +-- would silently blank every module's password. The hash has to outlive its own minting, which is +-- state the mesh did not need before and does now. +-- +-- Keyed by username, because the username is exactly what the composed file needs and what a +-- principal derives from its own identity. Nothing else about the user is here: **permissions are +-- not stored.** They are derived from what each module declares, every time the file is written +-- (ADR 0043) — a stored copy would be a second account of a user's authority, able to disagree +-- with the first, and the disagreement would be invisible until somebody compared a file with a +-- manifest. +-- +-- The hash and not the password. A file on a node's disk holds the hash, and so does this: a +-- credential recoverable from the mesh's store is one whose blast radius is the store's. +create table bus_user ( + username text primary key, + -- kind and what it names, so a user whose subject is gone can be found and removed: a module + -- unassigned, a node forgotten, a token spent. Recorded rather than parsed back out of the + -- username, because a name is for the server and a parser over it would be a second grammar. + kind text not null, + node text not null default '', + module text not null default '', + password_hash text not null, + minted_at timestamptz not null default now() +); + +-- Finding every user of one kind, and every user belonging to one node — which is what removing a +-- node, or composing after an assignment, asks. +create index bus_user_kind on bus_user (kind); +create index bus_user_node on bus_user (node) where node <> ''; -- 2.54.0 From ee1b8ffe24159f5ce629673c75fd14a85ff134e9 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 01:49:09 +0200 Subject: [PATCH 25/39] 1.7, second half: the user list read out of the mesh's records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The derivation had nothing feeding it. `BusRecords` reads what it needs — the machines, what each runs, every manifest, and which machines hold a live token — and turns it into the records the composer derives from. **A module's authority comes from its manifest, not from its assignment.** The assignment says where it runs; what it may say is what it declared. So the two are read together and the manifest decides, which is also why a seat's protocol is gathered across the whole catalogue rather than from one manifest: a seat is declared by one module and held by another, and that is the whole reason a seat exists. Three things checked against a real store, each a user that would be wrong in a way nothing reports: - A module assigned to a machine becomes a user with exactly the authority it declared, including the protocol of a seat some *other* module declared — a module granted nothing on a seat it was assigned to send to would fail on its first publish with an authorisation error that says nothing about a seat. - Only a machine holding a live token gets an enrolment user. One outliving its token is a right to join that nobody issued. - A module assigned and absent from the catalogue is refused rather than composed with an empty permission list. The catalogue already refuses to forget an assigned module, so this is the second line — and it earns its place there, because relying on another package's invariant is how a rule ends up enforced by nothing. People are left empty rather than guessed at: the account model is built and `operator issue` is not, so there is nobody to derive yet. --- internal/inventory/busrecords.go | 134 +++++++++++++++++++++ internal/inventory/busrecords_test.go | 164 ++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 internal/inventory/busrecords.go create mode 100644 internal/inventory/busrecords_test.go diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go new file mode 100644 index 0000000..61051f2 --- /dev/null +++ b/internal/inventory/busrecords.go @@ -0,0 +1,134 @@ +package inventory + +import ( + "context" + "fmt" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// What the bus's user list is derived from, read out of the mesh's records. +// +// The deriving itself is pure and lives in the broker package; this is the reading, and it is kept +// apart for the reason that package keeps its own types: a permission must be a function of what a +// module declared, and a query that decided anything would be a second place authority came from. + +// BusRecords is every fact the composer needs about who may reach the bus. +// +// **A module's authority comes from the manifest, not from the assignment.** The assignment says +// *where* it runs; what it may say is in what it declared, so the two are read together and the +// manifest is the one that decides. +func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) { + nodes, err := i.Nodes(ctx) + if err != nil { + return broker.Records{}, fmt.Errorf("cannot read the mesh's machines: %w", err) + } + declared, err := i.Catalogue(ctx) + if err != nil { + return broker.Records{}, fmt.Errorf("cannot read the catalogue: %w", err) + } + + // Every seat any module declares, by name, so a module's claim can be resolved to the protocol + // that seat promises. **Across the whole catalogue, not one manifest**: a seat is declared by + // one module and held by another, which is the whole reason a seat exists (ADR 0118). + seats := map[string]catalogue.SeatDeclaration{} + for _, m := range declared { + for _, s := range m.Seats { + seats[s.Name] = s + } + } + + out := broker.Records{Assigned: map[string][]broker.Declared{}, People: map[string][]string{}} + for _, n := range nodes { + out.Nodes = append(out.Nodes, n.Name) + modules, err := i.Assigned(ctx, n.Name) + if err != nil { + return broker.Records{}, fmt.Errorf("cannot read what %s runs: %w", n.Name, err) + } + for _, module := range modules { + m, known := declared[module] + if !known { + // Assigned and not in the catalogue. Said rather than composed with no authority: + // a user with an empty permission list is a module that starts, connects, and is + // refused by the server on its first publish — an authorisation error that says + // nothing about a missing manifest. + // + // **The catalogue refuses to forget an assigned module, so this is the second line + // and not the first.** It earns its place there anyway: relying on another + // package's invariant is how a rule ends up enforced by nothing. + return broker.Records{}, fmt.Errorf( + "%s is assigned to %s and is not in the catalogue, so what it may say cannot "+ + "be derived", module, n.Name) + } + out.Assigned[n.Name] = append(out.Assigned[n.Name], declaredFor(m, seats)) + } + } + + enrolling, err := i.NodesWithALiveToken(ctx) + if err != nil { + return broker.Records{}, err + } + out.Enrolling = enrolling + + // People are not recorded yet: the account model is built (design 25 §7's first item) and + // `operator issue` is not, so there is nobody to derive. Left empty rather than guessed at. + return out, nil +} + +// declaredFor is one module's manifest as the composer needs it: what it says about itself, and the +// protocol of every seat it holds or uses. +func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) broker.Declared { + d := broker.Declared{ + Module: m.Module, + Emits: m.Emits, + Consumes: m.Consumes, + // The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the + // facts a consumer needs to reach a provision, a different meaning under a similar word. + Serves: m.Tools, + } + for _, c := range m.Claims { + // A seat the mesh defines for itself declares no protocol, so holding one grants nothing + // here — which is right: those seats say who does a job, not who may say what. + if s, declaredSomewhere := seats[c.Name]; declaredSomewhere { + d.Holds = append(d.Holds, asSeat(s)) + } + } + for _, name := range m.Uses { + if s, declaredSomewhere := seats[name]; declaredSomewhere { + d.Uses = append(d.Uses, asSeat(s)) + } + } + return d +} + +func asSeat(s catalogue.SeatDeclaration) broker.Seat { + return broker.Seat{Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves} +} + +// NodesWithALiveToken is every machine holding a token that could still be presented — issued, not +// expired, not redeemed. +// +// **One enrolment user per such token** (design 25 §6): the inbox an answer goes to is scoped to the +// token, because an answer carries that machine's credentials sealed to it and a shared inbox is one +// machine able to read another's. +func (i *Inventory) NodesWithALiveToken(ctx context.Context) ([]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select distinct n.name + from enrolment_token t join node n on n.id = t.node + where t.redeemed is null and t.expires > now() + order by n.name`) + if err != nil { + return nil, fmt.Errorf("cannot read which machines hold a live token: %w", err) + } + defer rows.Close() + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} diff --git a/internal/inventory/busrecords_test.go b/internal/inventory/busrecords_test.go new file mode 100644 index 0000000..6c49263 --- /dev/null +++ b/internal/inventory/busrecords_test.go @@ -0,0 +1,164 @@ +package inventory + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// Reading the bus's user list out of the mesh's records, against a real store. +// +// What each of these is about is a user that would be **missing or wrong in a way nothing reports**: +// the server reads whatever file it is given, and a module whose user is absent fails on its first +// publish with an authorisation error that says nothing about a missing assignment. + +func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) { + t.Helper() + inv := ForTest(t) + ctx := context.Background() + for _, m := range manifests { + if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + return inv, ctx +} + +func theSeatDeclarer() catalogue.Manifest { + return catalogue.Manifest{ + Module: "telegram", Version: "1", + Seats: []catalogue.SeatDeclaration{{ + Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}, + }}, + Claims: []catalogue.Claim{{Name: "telegram-sender", Scope: catalogue.ScopeMesh}}, + } +} + +// A module assigned to a machine becomes a user with the authority its manifest declared — and the +// protocol of a seat declared by a *different* module, which is the whole reason a seat exists. +func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) { + shop := catalogue.Manifest{ + Module: "shop", Version: "1", + Emits: []string{"order.placed"}, Tools: []string{"price"}, + Uses: []string{"telegram-sender"}, + } + inv, ctx := aMeshWith(t, theSeatDeclarer(), shop) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + if err := inv.Assign(ctx, "one", "shop"); err != nil { + t.Fatal(err) + } + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + on := records.Assigned["one"] + if len(on) != 1 || on[0].Module != "shop" { + t.Fatalf("the machine's modules read as %+v", on) + } + if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" { + t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+ + "seat it was assigned to send to", on[0].Uses) + } + if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" { + t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+ + "serves nothing", on[0].Serves) + } + + // And it derives into a user the server would accept. + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var found bool + for _, u := range users { + if u.Username() != "one.shop" { + continue + } + found = true + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") || + !granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") || + !granted(perms.Subscribe, "mesh.mod.shop.tool.price") { + t.Fatalf("one.shop's authority is not what it declared: %+v", perms) + } + } + if !found { + t.Fatal("no user was derived for the assigned module") + } +} + +// A machine holding a live token gets an enrolment user; one whose token is spent or expired does +// not. **An enrolment user outliving its token is a right to join that nobody issued.** +func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) { + inv, ctx := aMeshWith(t) + for _, name := range []string{"live", "expired", "none"} { + if _, err := inv.AddNode(ctx, name); err != nil { + t.Fatal(err) + } + } + if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil { + t.Fatal(err) + } + // Briefly, then waited out: a token with no lifetime is refused at issue, which is the right + // refusal and leaves this as the way to have an expired one. + if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if strings.Join(records.Enrolling, ",") != "live" { + t.Fatalf("machines with a live token read as %v", records.Enrolling) + } +} + +// A module assigned and absent from the catalogue is refused rather than composed with no authority. +// +// **The catalogue refuses to forget an assigned module, so this is the second line and not the +// first** — and it earns its place there: relying on another package's invariant is how a rule ends +// up enforced by nothing. Checked against the derivation directly, because the situation cannot be +// reached through the store. +func TestAnAssignmentWithNoManifestDerivesNoAuthority(t *testing.T) { + // What BusRecords would have produced had it composed a ghost: a module with nothing declared. + users, err := broker.Users(broker.Records{ + Nodes: []string{"one"}, + Assigned: map[string][]broker.Declared{"one": {{Module: "ghost"}}}, + }) + if err != nil { + t.Fatal(err) + } + perms, err := broker.PermissionsFor(users[len(users)-1]) + if err != nil { + t.Fatal(err) + } + // Its inbox and its ack subject, and nothing it could say. That is a module which starts, + // connects, and is refused by the server on its first publish — an authorisation error that + // says nothing about a missing manifest, which is why BusRecords names it instead. + for _, p := range perms.Publish { + if strings.HasPrefix(p, "mesh.mod.ghost.event.") { + t.Fatalf("a module with no manifest was granted %s", p) + } + } +} + +func granted(all []string, one string) bool { + for _, s := range all { + if s == one { + return true + } + } + return false +} -- 2.54.0 From f8ab9f2dcf8b3e08c0ec7d377cdcc419e5c0bdb2 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 02:50:23 +0200 Subject: [PATCH 26/39] The mesh composes the accounts; the module owns its server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delivery question, decided. The alternative was a manifest field enumerating the server's ports, TLS paths and store directory so the controller could write a whole configuration file. That is wrong: those are properties of the container the module raises, they live in its image and its mounts, and the controller would have to be kept in step with a Dockerfile it never sees. So the mesh writes only what only the mesh knows — who may connect — and the module's own configuration includes it. `ComposeAccounts` is that file. A test says what must *not* be in it as plainly as what must: no port, no tls block, no store_dir. Each of those in the mesh's file is a value the controller would then own, and the module could no longer change its own image without the mesh agreeing. `bus-users` is where a module wants it written, and **asking is not enough to receive it**: the file holds every user's password hash, so a module that could ask for it could read every credential on the bus. The claim on `mesh-broker` authorises it, checked from the manifest alone. A holder with nothing composed is refused rather than given an empty file, for the reason a certificate is — a bus with no user list refuses every connection in the mesh and looks like a machine problem. Six claims checked against a running server before any of this was committed to, and two of them changed what got written: **An absolute include path is resolved relative to the including file's directory.** `include /etc/nats/accounts.conf` from /etc/nats-server/nats.conf makes the server look for /etc/nats-server/etc/nats/accounts.conf and refuse to start. So both files share one directory, and the module declares its own as a file resource beside the mesh's. **`verify: true` was refusing every connection in the mesh.** It makes the server demand a *client* certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and authenticates with the password the mesh minted, and so does a module's runtime. Every connection died at the TLS handshake before any password was looked at, with an error — "client didn't provide a certificate" — that reads as a fault in the client. Removed. TLS is still required; verify only decides whether client certificates are checked. The other four: a user in an included file authenticates, an unknown user is refused so the include is the whole authority rather than an addition, a publish outside a grant is refused, and rewriting the mesh's half alone makes a new user appear — noticed by the module's own watcher, with no signal from outside, and without dropping the connection the mesh already had. That last one is task 1.2's payoff, collected. --- internal/broker/nats.go | 44 +++++++++++++- internal/broker/testdata/composed.conf | 5 +- internal/broker/users_test.go | 47 +++++++++++++++ internal/catalogue/declaration.go | 36 ++++++++++++ internal/catalogue/declaration_test.go | 80 ++++++++++++++++++++++++++ internal/catalogue/manifest.go | 31 +++++++++- 6 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 internal/catalogue/declaration_test.go diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 83c0e87..159fbca 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -365,20 +365,60 @@ func Compose(s Server, principals []Principal) (string, error) { fmt.Fprintf(&b, "port: %d\n", s.ClientPort) fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort) + // **No `verify`, and it said `verify: true` until this configuration was run.** That setting + // makes the server demand a *client* certificate, and nothing in the mesh presents one: a host + // pins this server's exact certificate and authenticates with the password the mesh minted + // (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the + // mesh is refused at the TLS handshake before any password is looked at, and the error — + // "client didn't provide a certificate" — reads as a fault in the client. + // + // TLS is still required: the block is what requires it, and verify only decides whether client + // certificates are checked. b.WriteString("tls {\n") fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert) fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey) fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA) - b.WriteString(" verify: true\n") b.WriteString("}\n\n") b.WriteString("jetstream {\n") fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir) b.WriteString("}\n\n") + accounts, err := ComposeAccounts(sorted) + if err != nil { + return "", err + } + b.WriteString(accounts) + return b.String(), nil +} + +// ComposeAccounts is the accounts block alone — every user, and nothing about the server. +// +// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A +// server's ports, its TLS paths and its store directory are properties of the container the module +// raises: they live in its image and its mounts, and they change when it does. The controller has no +// business knowing them, and a controller that did would have to be kept in step with a Dockerfile +// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the +// module's own configuration includes it. +// +// Four things checked against a running server before this shape was committed to: a user in an +// included file authenticates; an unknown user is refused, so the include is the whole authority +// rather than an addition to something; a publish outside a user's grant is refused; and rewriting +// this file alone and signalling a reload makes a new user appear **without dropping the connection +// the mesh already has** — which is what makes every later account, permission or person change cost +// nothing (task 1.2's payoff). +func ComposeAccounts(principals []Principal) (string, error) { + sorted := append([]Principal(nil), principals...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() }) + + var b strings.Builder + b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n") + b.WriteString("# composition overwrites it. Permissions are derived from what each module\n") + b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n") + // One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is // one space (design 25 §4). The cost of that — that permissions are the only isolation — is - // paid above, in the scoping of every inbox and every ack subject. + // paid in the scoping of every inbox and every ack subject. b.WriteString("accounts {\n MESH {\n users = [\n") for _, p := range sorted { perms, err := PermissionsFor(p) diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 489435a..dd7da95 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -9,13 +9,16 @@ tls { cert_file: "/tls/tls.crt" key_file: "/tls/tls.key" ca_file: "/tls/ca.crt" - verify: true } jetstream { store_dir: "/data" } +# The mesh's users, composed by the controller. Do not edit: the next +# composition overwrites it. Permissions are derived from what each module +# declares and nothing else (novox/hq ADR 0043, design 29 §2). + accounts { MESH { users = [ diff --git a/internal/broker/users_test.go b/internal/broker/users_test.go index 5787b9e..8f40e1b 100644 --- a/internal/broker/users_test.go +++ b/internal/broker/users_test.go @@ -154,3 +154,50 @@ func TestRecordsComposeIntoAFile(t *testing.T) { } } } + +// The accounts block alone is what the mesh writes, and it holds nothing about the server. +// +// **The split is the whole design decision** (ComposeAccounts): ports, TLS paths and a store +// directory are properties of the container the module raises, and a controller that wrote them +// would have to be kept in step with a Dockerfile it never sees. So this test says what must not be +// in the file as plainly as what must. +func TestWhatTheMeshWritesIsUsersAndNothingAboutTheServer(t *testing.T) { + users, err := Users(someRecords()) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for _, u := range users { + hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22) + } + filled, missing := WithPasswords(users, hashes) + if len(missing) != 0 { + t.Fatalf("users with no password: %v", missing) + } + got, err := ComposeAccounts(filled) + if err != nil { + t.Fatal(err) + } + + for _, want := range []string{"accounts {", `user: "controller"`, `user: "one.telegram"`} { + if !strings.Contains(got, want) { + t.Errorf("the accounts file does not contain %s", want) + } + } + // None of the server's own settings. Each of these in the mesh's file is a value the controller + // would then own, and the module could no longer change its own image without the mesh agreeing. + for _, absent := range []string{"port:", "http:", "jetstream", "tls {", "store_dir", "cert_file"} { + if strings.Contains(got, absent) { + t.Errorf("the accounts file contains %q, which belongs to the module that raises the "+ + "server, not to the mesh", absent) + } + } +} + +// A user with no password is refused here too, not only by the whole-file composition: this is the +// function the controller actually calls, and a user without a password is a user anybody is. +func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) { + if _, err := ComposeAccounts([]Principal{{Kind: KindController}}); err == nil { + t.Fatal("a user with no password hash was written") + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index ad33e90..f78f661 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -97,6 +97,13 @@ type Rendering struct { // compose it a second time. Suffix string + // BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on + // every other node, and on this one until the controller has composed it. + // + // **Only the users, never the server's own settings**: those are the module's, in its image and + // its mounts (Manifest.BusUsers). + BusUsers string + // Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when // nothing on this node keeps them, or the mesh has no operator key. Kept *KeptExport @@ -359,6 +366,35 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri }) } } + if m.BusUsers != "" { + // **The claim authorises it, not the field.** This file holds every user's password + // hash, so a module that could ask for it could read every credential on the bus. + // Checked from this manifest alone, which is the cheapest check there is: whether some + // other module also claims the seat is resolution's business elsewhere, and one holder + // mesh-wide is already guaranteed. + if !m.ClaimsSeat("mesh-broker") { + return nil, fmt.Errorf( + "%s asks for the mesh's user list and does not claim mesh-broker. That file "+ + "holds every user's password hash, so the seat is what authorises it", + m.Module) + } + if with.BusUsers == "" { + // Asked for and not composed. Refused rather than skipped, for the reason a + // certificate is: a bus with no user list refuses every connection in the mesh, and + // an empty file would look like a configuration problem on the machine. + return nil, fmt.Errorf( + "%s holds mesh-broker and the mesh composed no user list, so the bus would "+ + "refuse every connection", m.Module) + } + first = append(first, map[string]any{ + "id": BusUsersID(), "type": "file", "path": m.BusUsers, + "content": with.BusUsers, + // Readable by the server and nothing else. Hashes rather than passwords, so this is + // not a set of working credentials — but a list of every user in the mesh is worth + // keeping to the one process that needs it. + "mode": "0600", + }) + } for _, name := range sortedKeys(m.OwnSecrets) { sealed := with.Needed[m.Module][name] if sealed == "" { diff --git a/internal/catalogue/declaration_test.go b/internal/catalogue/declaration_test.go new file mode 100644 index 0000000..23b1caa --- /dev/null +++ b/internal/catalogue/declaration_test.go @@ -0,0 +1,80 @@ +package catalogue + +import "testing" + +// The mesh's user list reaches the module holding the bus, and nothing else. +// +// Three refusals and one delivery, because each of the refusals would be silent in a different way: +// a module that asked and was given it could read every credential on the bus; a bus given an empty +// file refuses every connection in the mesh and looks like a machine problem; and a bus that never +// asked gets nothing rather than a file it does not read. +func TestTheMeshsUserListGoesOnlyToTheModuleHoldingTheBus(t *testing.T) { + theBus := func() Manifest { + return Manifest{ + Module: "nats", Version: "1", + Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}, + BusUsers: "/var/lib/nats-module/conf/accounts.conf", + Resources: []map[string]any{}, + } + } + + on := func(t *testing.T, m Manifest, with Rendering) ([]map[string]any, error) { + t.Helper() + return Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(with) + } + + t.Run("the holder is given it", func(t *testing.T) { + resources, err := on(t, theBus(), Rendering{BusUsers: "accounts { MESH { users = [] } }"}) + if err != nil { + t.Fatal(err) + } + // Prefixed with the module it came from, like every resource: two modules may reasonably + // both call something "config", and without the prefix the second would silently replace + // the first. + var found map[string]any + for _, r := range resources { + if r["id"] == "nats."+BusUsersID() { + found = r + } + } + if found == nil { + t.Fatalf("the bus was given no user list: %+v", resources) + } + if found["path"] != "/var/lib/nats-module/conf/accounts.conf" { + t.Errorf("written to %v rather than where the module asked", found["path"]) + } + if found["mode"] != "0600" { + t.Errorf("mode %v: a list of every user in the mesh belongs to the one process that "+ + "needs it", found["mode"]) + } + }) + + t.Run("a module that does not claim the seat is refused", func(t *testing.T) { + m := theBus() + m.Claims = nil + if _, err := on(t, m, Rendering{BusUsers: "accounts {}"}); err == nil { + t.Fatal("a module that claims nothing was handed every user's password hash") + } + }) + + t.Run("the holder with nothing composed is refused", func(t *testing.T) { + if _, err := on(t, theBus(), Rendering{}); err == nil { + t.Fatal("the bus was given an empty user list, so it would refuse every connection in " + + "the mesh and look like a machine problem") + } + }) + + t.Run("a module that did not ask gets nothing", func(t *testing.T) { + m := theBus() + m.BusUsers = "" + resources, err := on(t, m, Rendering{BusUsers: "accounts {}"}) + if err != nil { + t.Fatal(err) + } + for _, r := range resources { + if r["id"] == "nats."+BusUsersID() { + t.Fatal("a module that asked for no user list was given one") + } + } + }) +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 7115e1a..b874536 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -423,6 +423,20 @@ type Manifest struct { // A directory rather than one document for the same reason as above: each value is sealed // separately and the mesh cannot open any of them to build a list. Grants map[string]string `json:"grants,omitempty"` + + // BusUsers is where this module wants the mesh's user list written, and it is only ever + // answered for the module holding `mesh-broker`. + // + // **The mesh writes who may connect; the module owns everything else about its server** + // (novox/hq design 25 §4, task 1.7). Ports, TLS paths and a store directory live in this + // module's image and its mounts and change when it does, so the module's own configuration + // carries them and includes this file. A controller that wrote the whole configuration would + // have to be kept in step with a Dockerfile it never sees. + // + // **Asking for it is not enough to receive it.** This file holds every user's password hash, so + // a module that could ask for it could read every credential on the bus — and the claim on + // `mesh-broker` is what authorises it, checked from this manifest alone. + BusUsers string `json:"bus-users,omitempty"` } // Build says how to produce this module's artifacts from its source. @@ -641,7 +655,22 @@ type Certificate struct { // CertificateID and AuthorityID are the resource identities of what the mesh issued. func CertificateID() string { return "certificate" } -func AuthorityID() string { return "certificate-authority" } + +// ClaimsSeat says whether this manifest claims one named seat. +func (m Manifest) ClaimsSeat(seat string) bool { + for _, c := range m.Claims { + if c.Name == seat { + return true + } + } + return false +} + +// BusUsersID names the mesh's composed user list, so it is the same resource across every +// declaration and a change to it is an update rather than a second file beside the old one — which +// on a bus reading a directory would be two account lists, and the server would take both. +func BusUsersID() string { return "bus-users" } +func AuthorityID() string { return "certificate-authority" } // FilteringID names the computed rule set, so it is the same resource across every declaration // and a change to it is an update rather than an addition beside the old one. -- 2.54.0 From 4de10e32e370830e9e4e30b8b7243c2088e80e8a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 02:59:05 +0200 Subject: [PATCH 27/39] The bus's objects are raised on every start, and one switch says which bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of 1.7's three remaining pieces. **Raised on every start, not created once at genesis.** A stream somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was replaced all have records and no objects — and a node whose consumer is missing hears nothing while everything else about it looks correct. The order is not a preference: a consumer on a stream that does not exist is refused *naming the stream*, so somebody reading that refusal goes looking for a deletion instead of a reversed pair of lines. Pinned by a test, along with the one thing about seats that reads like an omission and is not — a seat's work queue is asserted whether or not anybody holds it, because work queues until a holder appears, so installing the module a week later flushes the backlog instead of having lost it. Against a real server: every object accepted, asserting twice changes nothing (a start that failed the second time is a controller that cannot restart), a machine joining an already-raised bus is accepted, each node's consumer is bound to its own declaration subject and no other's, and CONTROL does not dead-letter — because the store window's bound belongs to the controller and a server that gave up first would discard the push the stream exists to protect. **Which bus this mesh is on is one fact, read in one place.** Every seam the change went behind ships both implementations; this is what the rollout flips. Being told about both is refused at start rather than warned about: a mesh half on each is one where a declaration goes out on one bus and the report comes back on the other, and every component logs success while it happens — ADR 0074's failure arriving through configuration instead of through code. The refusal names both variables and says which to unset, because whoever reads it has to choose and the wrong choice is a rollout half done. --- cmd/mesh-controller/push.go | 52 +++++++++++++++ internal/broker/onnats.go | 58 ++++++++++++++++ internal/broker/onnats_test.go | 40 +++++++++++ internal/broker/raise.go | 73 ++++++++++++++++++++ internal/broker/raise_live_test.go | 104 +++++++++++++++++++++++++++++ internal/broker/streams_test.go | 90 +++++++++++++++++++++++++ 6 files changed, 417 insertions(+) create mode 100644 internal/broker/onnats.go create mode 100644 internal/broker/onnats_test.go create mode 100644 internal/broker/raise.go create mode 100644 internal/broker/raise_live_test.go diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 2ee6875..d02e40c 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -77,12 +77,33 @@ func serve(ctx context.Context) error { "reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar) } + // **Which bus this mesh is on, read once** (novox/hq ADR 0116 step 5). Both clients ship; both + // being live is refused, because a mesh half on each is one where a declaration goes out on one + // and the report comes back on the other, and every component logs success while it happens. + busAddress, onNATS, err := broker.OnNATS() + if err != nil { + return err + } + if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil { + return err + } + work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known} server, err := link.Connect(work, work) if err != nil { return err } defer server.Close() + + // The bus's own objects, asserted on every start. **Not created once at genesis**: a stream + // somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was + // replaced all have records and no objects — and a node whose consumer is missing hears nothing + // while everything else about it looks correct. + if onNATS { + if err := raiseTheBus(ctx, inv, busAddress); err != nil { + return err + } + } // And build results nobody was waiting for. A build triggered any other way than `build` // would otherwise be reported into the void, which is the same as not reporting it. server.Records(builds{inv}) @@ -664,3 +685,34 @@ func wouldSend(ctx context.Context, open *stores, } return out, nil } + +// raiseTheBus asserts the streams and consumers the mesh's own traffic needs. +// +// **Every start, and it says what it did.** The objects are the mesh's, created by nothing else — +// the controller is their only writer (design 25 §3) — so a mesh that came up without them is one +// where nodes connect, authenticate, and hear nothing. Said rather than silent for the reason the +// first line of `serve` is said: a log that is quiet on success and loud on failure reads as broken +// when it is working. +func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) error { + js, err := broker.Dial(address) + if err != nil { + return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w", + address, err) + } + defer js.Close() + + nodes, err := inv.Nodes(ctx) + if err != nil { + return err + } + names := make([]string, 0, len(nodes)) + for _, n := range nodes { + names = append(names, n.Name) + } + if err := broker.Raise(js, names); err != nil { + return err + } + fmt.Printf("the bus at %s has its streams, and %d machine(s) can hear a declaration\n", + address, len(names)) + return nil +} diff --git a/internal/broker/onnats.go b/internal/broker/onnats.go new file mode 100644 index 0000000..f6efa54 --- /dev/null +++ b/internal/broker/onnats.go @@ -0,0 +1,58 @@ +package broker + +import ( + "fmt" + "strings" + + "github.com/novox/mesh-controller/internal/envfile" +) + +// Whether this mesh's own traffic is on the bus being built. +// +// **One switch, read in one place** (novox/hq ADR 0116 step 5). Every seam the bus change went +// behind ships both implementations, and until the rollout every one of them chooses the bus the +// mesh runs on today. This is what the rollout flips, and it is deliberately a single fact rather +// than a fact per component: a controller whose outbound is on one bus and whose inbound is on the +// other is a mesh that hears nothing, and no test of either half would catch it. + +// NATSVar is where the controller finds the bus being built. Unset is the ordinary case and means +// the mesh runs on the bus it has always run on. +const NATSVar = "MESH_BUS_NATS" + +// OnNATS is the address of the bus being built, and whether the mesh is on it. +// +// Read from the node's own settings rather than baked in, for the reason the broker's address is +// (novox/hq 04-ISSUES/102): an address recorded once does not follow a node's ports. +func OnNATS() (address string, on bool, err error) { + address, err = envfile.Placed(NATSVar) + if err != nil { + return "", false, err + } + address = strings.TrimSpace(address) + if address == "" { + return "", false, nil + } + return address, true, nil +} + +// MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic. +// +// **Both clients ship and that is the point; both being live is not.** The rollout moves every node +// at once (ADR 0116 step 5): a mesh half on each is one where a declaration goes out on one bus and +// the report comes back on the other, and nothing anywhere says so — every component would log +// success. Refused at start, where it can be said in one sentence. +func MustBeOneBus(amqp, nats string) error { + if strings.TrimSpace(amqp) != "" && strings.TrimSpace(nats) != "" { + return fmt.Errorf( + "this control plane is told about both buses (%s and %s) and can only be on one. A mesh "+ + "half on each is one where a declaration goes out on one and the report comes back "+ + "on the other, and every component reports success while it happens. The rollout "+ + "moves every node at once: unset %s to stay, or unset %s to move", + AMQPVarName, NATSVar, NATSVar, AMQPVarName) + } + return nil +} + +// AMQPVarName is the variable naming the bus the mesh runs on today. Named here rather than +// imported from the link package, for the one direction of dependency. +const AMQPVarName = "MESH_BROKER_AMQP" diff --git a/internal/broker/onnats_test.go b/internal/broker/onnats_test.go new file mode 100644 index 0000000..19e78e9 --- /dev/null +++ b/internal/broker/onnats_test.go @@ -0,0 +1,40 @@ +package broker + +import ( + "strings" + "testing" +) + +// Which bus the mesh is on is one fact, and being told about both is refused. +// +// **Not a warning.** A mesh half on each bus is one where a declaration goes out on one and the +// report comes back on the other, and every component reports success while it happens — which is +// the exact failure ADR 0074 exists to catch, arriving through configuration instead of through code. +func TestBeingToldAboutBothBusesIsRefused(t *testing.T) { + err := MustBeOneBus("amqps://broker:5671/", "nats://bus:4222") + if err == nil { + t.Fatal("a control plane told about both buses was allowed to start") + } + // The remedy is in the words, because whoever reads this has to choose one and the wrong choice + // is a rollout half done. + for _, want := range []string{AMQPVarName, NATSVar, "unset"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not mention %s: %v", want, err) + } + } +} + +// One bus, or none, is ordinary. None is a control plane that publishes nothing and holds records, +// which several of its own commands are. +func TestOneBusOrNeitherIsAllowed(t *testing.T) { + for _, c := range []struct{ what, amqp, nats string }{ + {"the bus the mesh runs on today", "amqps://broker:5671/", ""}, + {"the bus being built", "", "nats://bus:4222"}, + {"neither", "", ""}, + {"neither, with whitespace for an address", " ", "\t"}, + } { + if err := MustBeOneBus(c.amqp, c.nats); err != nil { + t.Errorf("%s was refused: %v", c.what, err) + } + } +} diff --git a/internal/broker/raise.go b/internal/broker/raise.go new file mode 100644 index 0000000..6f3d15a --- /dev/null +++ b/internal/broker/raise.go @@ -0,0 +1,73 @@ +package broker + +import "fmt" + +// Bringing the bus's own objects into being, in the one order that works. +// +// **Asserted on every start rather than created once at genesis.** A stream somebody deleted, a mesh +// raised from a restored backup, or a bus whose data directory was replaced all have records and no +// objects — and a node whose consumer is missing hears nothing while everything else about it looks +// correct. Idempotence is the whole requirement, and the parts are already idempotent; this is the +// order they have to be asked in. + +// Raiser is everything asserting the bus's objects needs of a connection to it. +type Raiser interface { + Asserter + Ensurer +} + +// Raise asserts the mesh's streams, the controller's own consumers, and one consumer per node. +// +// **The order is not a preference.** A consumer on a stream that does not exist is refused, and the +// refusal names the stream rather than the order — so somebody reading it goes looking for a deleted +// stream instead of a reversed pair of lines. Nodes last, because the one a node reads lives on a +// stream the mesh's own set defines. +func Raise(r Raiser, nodes []string) error { + if err := AssertMeshStreams(r); err != nil { + return err + } + if err := AssertMeshConsumers(r); err != nil { + return err + } + if err := AssertNodeConsumers(r, nodes); err != nil { + return err + } + return nil +} + +// RaiseSeats asserts one work queue per declared seat, and the worker of whoever holds it. +// +// Separate from Raise because it is answered by a different question: the mesh's own objects exist +// because the mesh does, and a seat's exist because a module declaring one was registered. Kept +// beside it so the order is visible — a holder's worker needs the seat's stream, and a seat's stream +// needs nothing. +func RaiseSeats(r Raiser, seats []DeclaredSeat, holders map[string]Holder) error { + for _, s := range SeatStreams(seats) { + if err := r.EnsureStream(s); err != nil { + return fmt.Errorf("asserting the work queue for %s: %w", s.Name, err) + } + } + for _, s := range seats { + h, held := holders[s.Name] + if !held { + // **The stream exists and the consumer does not, on purpose.** Work queues until a + // holder appears, so installing the module a week after something started sending to it + // flushes the backlog instead of having lost it. + continue + } + c, needed := HolderConsumerFor(h.Node, h.Module, s) + if !needed { + continue + } + if err := r.EnsureConsumer(c); err != nil { + return fmt.Errorf("asserting how %s on %s works %s: %w", h.Module, h.Node, s.Name, err) + } + } + return nil +} + +// Holder is which module on which machine holds a seat. +type Holder struct { + Node string + Module string +} diff --git a/internal/broker/raise_live_test.go b/internal/broker/raise_live_test.go new file mode 100644 index 0000000..bf468a0 --- /dev/null +++ b/internal/broker/raise_live_test.go @@ -0,0 +1,104 @@ +package broker + +import ( + "os" + "testing" + + "github.com/nats-io/nats.go" +) + +// Raising the bus's objects against a real server. +// +// The pure tests above say what is asked for and in what order. Only a server can say whether it +// accepts them — and two of these are claims about the server's own behaviour that nothing else +// could answer: that asserting twice changes nothing, and that a consumer really is bound to the one +// subject its node is allowed to read. +// +// docker run -d --rm --name t -p 14227:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14227 go test ./internal/broker/ -run TestRaising + +func aLiveBus(t *testing.T) *JetStream { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + // Deleted before, so what this test asserts is what it finds — and after, so the next test does + // not inherit it. Deleting a stream takes its consumers with it, which is why this is enough. + clear := func() { + for _, s := range MeshStreams() { + _ = js.Context().DeleteStream(s.Name) + } + } + clear() + t.Cleanup(clear) + return js +} + +// Every object the mesh's own traffic needs, accepted by a real server, and asserting again changes +// nothing — which is the whole requirement, because this runs on every start. +func TestRaisingTheBusIsAcceptedAndIdempotent(t *testing.T) { + js := aLiveBus(t) + + if err := Raise(js, []string{"anchor", "laptop"}); err != nil { + t.Fatalf("a real server refused the mesh's own objects: %v", err) + } + // Twice, with nothing in between. A start that failed the second time is a controller that + // cannot restart. + if err := Raise(js, []string{"anchor", "laptop"}); err != nil { + t.Fatalf("asserting the bus's objects a second time failed, so a restart would: %v", err) + } + // And again with a machine that was not there before, which is what enrolling one is. + if err := Raise(js, []string{"anchor", "laptop", "workstation"}); err != nil { + t.Fatalf("a machine joining an already-raised bus was refused: %v", err) + } + + for _, s := range MeshStreams() { + if _, err := js.Context().StreamInfo(s.Name); err != nil { + t.Errorf("stream %s is not there: %v", s.Name, err) + } + } + for _, c := range MeshConsumers() { + if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil { + t.Errorf("the controller's consumer on %s is not there: %v", c.Stream, err) + } + } + for _, node := range []string{"anchor", "laptop", "workstation"} { + info, err := js.Context().ConsumerInfo("NODES", node) + if err != nil { + t.Errorf("%s has no way to hear its declaration: %v", node, err) + continue + } + // **Its own subject and no other node's.** A consumer filtered on anything wider is a node + // reading another machine's declaration, and its own ack grant would not cover it either. + if info.Config.FilterSubject != "mesh.node."+node+".declare" { + t.Errorf("%s's consumer reads %q", node, info.Config.FilterSubject) + } + if info.Config.AckPolicy != nats.AckExplicitPolicy { + t.Errorf("%s's consumer acknowledges on delivery, so a declaration it died applying is "+ + "never sent again", node) + } + } +} + +// The store window needs unlimited redelivery on CONTROL: the bound belongs to the controller, and a +// server that dead-lettered first would discard the push the stream exists to protect. +func TestTheControlConsumerDoesNotDeadLetterBeforeTheControllerGivesUp(t *testing.T) { + js := aLiveBus(t) + if err := Raise(js, nil); err != nil { + t.Fatal(err) + } + info, err := js.Context().ConsumerInfo("CONTROL", ControllerName) + if err != nil { + t.Fatal(err) + } + if info.Config.MaxDeliver > 0 { + t.Fatalf("max-deliver is %d: a push held through a store restart would be dead-lettered "+ + "before the controller finished deciding about it", info.Config.MaxDeliver) + } +} diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go index bd41654..dc70bb8 100644 --- a/internal/broker/streams_test.go +++ b/internal/broker/streams_test.go @@ -144,3 +144,93 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) { } } } + +// The order the bus's objects are asserted in, because getting it wrong is a refusal that names the +// wrong thing: a consumer on a stream that does not exist is refused naming the *stream*, so +// somebody reading it goes looking for a deletion instead of a reversed pair of lines. +func TestTheBusesObjectsAreAssertedStreamsBeforeConsumers(t *testing.T) { + r := &recording{} + if err := Raise(r, []string{"anchor", "laptop"}); err != nil { + t.Fatal(err) + } + + // Every stream before every consumer. + firstConsumer := -1 + for i, step := range r.steps { + if strings.HasPrefix(step, "consumer ") && firstConsumer < 0 { + firstConsumer = i + } + if strings.HasPrefix(step, "stream ") && firstConsumer >= 0 { + t.Fatalf("a stream was asserted after a consumer: %v", r.steps) + } + } + if firstConsumer < 0 { + t.Fatalf("no consumer was asserted: %v", r.steps) + } + + // And every node got one, named after it — without which that node hears nothing while + // everything else about it looks correct. + for _, node := range []string{"anchor", "laptop"} { + if !containsStep(r.steps, "consumer NODES/"+node) { + t.Errorf("%s was given no way to hear its declaration: %v", node, r.steps) + } + } + // And the controller its own, on both streams it reads. + for _, want := range []string{"consumer CONTROL/controller", "consumer EVENTS/controller"} { + if !containsStep(r.steps, want) { + t.Errorf("the controller is missing %s: %v", want, r.steps) + } + } +} + +// A seat's work queue is asserted whether or not anybody holds it; the holder's worker only when +// somebody does. **The stream without the consumer is the point**: work queues until a holder +// appears, so installing the module later flushes the backlog instead of having lost it. +func TestASeatsQueueExistsBeforeItsHolderDoes(t *testing.T) { + seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}} + + unheld := &recording{} + if err := RaiseSeats(unheld, seats, nil); err != nil { + t.Fatal(err) + } + if !containsStep(unheld.steps, "stream SEAT_TELEGRAM_SENDER") { + t.Fatalf("a declared seat got no work queue: %v", unheld.steps) + } + for _, step := range unheld.steps { + if strings.HasPrefix(step, "consumer ") { + t.Fatalf("a seat nobody holds got a worker: %v", unheld.steps) + } + } + + held := &recording{} + if err := RaiseSeats(held, seats, map[string]Holder{ + "telegram-sender": {Node: "anchor", Module: "telegram"}, + }); err != nil { + t.Fatal(err) + } + if !containsStep(held.steps, "consumer SEAT_TELEGRAM_SENDER/SEAT_TELEGRAM_SENDER_worker") { + t.Fatalf("the seat's holder got no worker: %v", held.steps) + } +} + +// recording is a connection to the bus that writes down what it was asked for. +type recording struct{ steps []string } + +func (r *recording) EnsureStream(s Stream) error { + r.steps = append(r.steps, "stream "+s.Name) + return nil +} + +func (r *recording) EnsureConsumer(c Consumer) error { + r.steps = append(r.steps, "consumer "+c.Stream+"/"+c.Name) + return nil +} + +func containsStep(steps []string, want string) bool { + for _, s := range steps { + if s == want { + return true + } + } + return false +} -- 2.54.0 From eb72ec36ba52cf4307b1f3cdbe13699efd20c95e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 03:19:41 +0200 Subject: [PATCH 28/39] 1.7 finished: minting, the file delivered, and a test flake I caused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **First, a correction: the previous commit went in on a false check.** Its message says the suite passed; it did not. The check piped `go test` through a filter that swallowed the failures and then printed "green" regardless. Two tests were failing when 4de10e3 landed. What was failing was my own doing. Purging the streams instead of deleting them (4de10e3) left the *consumers* behind, because deleting a stream takes its consumers with it and purging does not. A durable push consumer surviving between tests keeps pushing to a delivery subject the previous test's subscription has gone from: the messages count as delivered, go nowhere, and the next test waits out its timeout for an announcement the server believes it already sent. Consumers are now removed with the purge. Five consecutive clean runs. `-p 1` stays, because two packages asserting and deleting the same fixed-name objects on one bus is a real race — but its comment said the cause I had guessed and not the one I found, so it now says the right thing. **And delivery was not finished when I said it was.** Nothing filled `Rendering.BusUsers`, so the composed file would never have reached a node. `composeBusUsers` closes it: composed per push for the machine holding `mesh-broker`, never kept, because the list is a function of the mesh's records and a stored copy could disagree with them while both looked consistent. A user with no credential is left out and named rather than written as a user without a password — an ordinary situation with an obvious remedy — but a file with no users at all is refused, because that bus would refuse every connection in the mesh. **Minting, on both halves.** A node at enrolment and a module at `module issue`. Three things differ from a management call and each is the point of the move: the credential is minted into the mesh's records and becomes usable at the next composition, so no server need be reachable; the password travels beside the address rather than inside it, because a credential embedded in a URL leaks into every log line that prints a connection; and a module's durable consumer is derived from what it declared rather than named, so it cannot ask for delivery of something it did not say it consumes. A node reconnecting may be refused until that composition reaches the machine running the bus. That is what the host's reconnect backoff is for and it is survivable by design; waiting for the push would hold an enrolment open for as long as a declaration takes to apply. Tested that the switch is a switch: a node enrolling on one bus comes away with a credential for that bus and none for the other, because one that held both could be half-moved and nothing would say which half. --- Makefile | 12 ++- cmd/mesh-controller/modules.go | 89 +++++++++++++++++ cmd/mesh-controller/plan.go | 71 ++++++++++++++ cmd/mesh-controller/push.go | 3 +- internal/broker/raise_live_test.go | 15 ++- internal/link/enrol_bus_credential_test.go | 107 +++++++++++++++++++++ internal/link/enrolment.go | 37 ++++++- internal/link/receive_nats_test.go | 29 ++++-- 8 files changed, 340 insertions(+), 23 deletions(-) create mode 100644 internal/link/enrol_bus_credential_test.go diff --git a/Makefile b/Makefile index 2e21062..a41e6d7 100644 --- a/Makefile +++ b/Makefile @@ -86,13 +86,19 @@ proxy-image: # The whole gate. Raises a database, runs everything against it, and takes it down again -- # including when the tests fail, which is why the teardown is not conditional. +# +# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on +# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names, +# because those names are the mesh's and a test cannot choose others. Two packages doing that at once +# is one deleting a consumer the other is reading through, and the failure lands in whichever test +# was reading, as "no response from stream". That reads as a bug in the code under test. check: fmt vet postgres - @go test ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status + @go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status # Without a database the live tests skip rather than fail, so this is the honest subset and not -# the gate. +# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not. test: - go test ./... + go test -p 1 ./... vet: go vet ./... diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 1d0d72d..fdb719f 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -257,6 +257,21 @@ func moduleCommand(ctx context.Context, args []string) error { return err } + // Which bus this mesh is on. A module gets a credential for exactly one, and the two are + // made in entirely different ways: on the bus the mesh runs on today an account is a + // management call, and on the bus being built it is a row the next composition writes into + // the server's user list (novox/hq design 25 §4). + busAddress, onNATS, err := broker.OnNATS() + if err != nil { + return err + } + if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil { + return err + } + if onNATS { + return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress) + } + management, err := broker.ManagementFromEnvironment() if err != nil { return err @@ -567,3 +582,77 @@ func mayIssue(m catalogue.Manifest) error { } return nil } + +// issueOnTheNewBus gives an assigned module its credential on the bus being built. +// +// **Three things differ from a management call, and each is the point of the move.** The credential +// is minted into the mesh's records and becomes usable at the next composition, so there is no +// server to be reachable for this to work. The password travels beside the address rather than inside +// it, because the runtime's contract already separates them and a credential embedded in a URL is one +// that leaks into every log line that prints a connection. And the module's durable consumer is +// derived from what it declared rather than declared by name, so a module cannot ask for delivery of +// something it did not say it consumes. +func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, + node, busAddress string) error { + + user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username() + password, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module, + }) + if err != nil { + return err + } + + // Where the module is told to find the bus, and what certificate it must present. The same pair + // a node is told, for the same reason: a mesh's bus presents its own certificate, in no public + // trust store, so an address alone fails at TLS. + known, err := broker.FromEnvironment() + if err != nil { + return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err) + } + reachable, err := brokerReachableAt(ctx, inv, known, node) + if err != nil { + return err + } + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node"` + Module string `json:"module"` + User string `json:"user"` + Password string `json:"password"` + }{ + URL: "nats://" + reachable, Fingerprint: known.Fingerprint, + Node: node, Module: m.Module, User: user, Password: password, + }) + if err != nil { + return err + } + if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil { + return err + } + + // And how it hears what it consumes. Derived from its declaration, and only when it declared + // something: a module that consumes nothing needs no consumer, and creating one would be a + // durable subscription nobody reads. + if consumer, needed := broker.ConsumerFor(broker.Principal{ + Kind: broker.KindModule, Node: node, Module: m.Module, + Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools, + }); needed { + js, err := broker.Dial(busAddress) + if err != nil { + return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ + "how %s hears what it consumes: %w", m.Module, err) + } + defer js.Close() + if err := js.EnsureConsumer(consumer); err != nil { + return err + } + } + + fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module) + fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node) + fmt.Printf(" and it works once the bus has been told: the user list is composed into the " + + "machine holding mesh-broker\n") + return nil +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index e744e2d..017f5b8 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -585,12 +585,19 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + // The bus's user list, for the machine that runs the bus. Composed per push rather than kept, + // because it is a function of the mesh's records and a kept copy could disagree with them. + busUsers, err := composeBusUsers(ctx, inv, plan.Modules) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted, Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built, + BusUsers: busUsers, }, record, nil } @@ -1135,3 +1142,67 @@ func portsOn( } return out, nil } + +// composeBusUsers is the bus's user list, for a push to the machine that runs the bus. +// +// Empty for every other machine, and for every machine while the mesh is on the bus it runs on +// today — where accounts are a management call and there is no file to write. +// +// **Composed on each push, never kept.** The list is a function of the mesh's records (who exists, +// what runs where, what each declares), and a stored copy would be a second account of who may reach +// the bus, able to disagree with the records while both looked internally consistent (ADR 0043). +// +// A user the mesh has never minted a password for is **left out and said**, not written as a user +// without one — the composer refuses that, because a user with no password is a user anybody is. That +// is an ordinary situation with an obvious remedy (`module issue`, or enrolling), so the push carries +// the rest rather than failing: a bus that is missing one module's user is a mesh where that module +// cannot connect, and a bus with no file at all is a mesh where nothing can. +func composeBusUsers(ctx context.Context, inv *inventory.Inventory, + onThisNode []catalogue.Manifest) (string, error) { + + _, onNATS, err := broker.OnNATS() + if err != nil || !onNATS { + return "", err + } + // Only for the machine holding the bus. Asked of what this push resolves to rather than of the + // seat's holder mesh-wide: the file is a resource of that module, so the question is whether it + // is here. + holdsTheBus := false + for _, m := range onThisNode { + if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { + holdsTheBus = true + } + } + if !holdsTheBus { + return "", nil + } + + records, err := inv.BusRecords(ctx) + if err != nil { + return "", err + } + users, err := broker.Users(records) + if err != nil { + return "", err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return "", err + } + hashes := make(map[string]string, len(kept)) + for name, u := range kept { + hashes[name] = u.PasswordHash + } + filled, missing := broker.WithPasswords(users, hashes) + if len(missing) > 0 { + fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+ + "for: %s. Each is a user that cannot connect until one is issued\n", + len(missing), strings.Join(missing, ", ")) + } + if len(filled) == 0 { + return "", fmt.Errorf( + "this machine runs the bus and not one user has a credential, so the composed list " + + "would refuse every connection in the mesh") + } + return broker.ComposeAccounts(filled) +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index d02e40c..1cafa57 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -88,7 +88,8 @@ func serve(ctx context.Context) error { return err } - work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known} + work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known, + OnNATS: onNATS} server, err := link.Connect(work, work) if err != nil { return err diff --git a/internal/broker/raise_live_test.go b/internal/broker/raise_live_test.go index bf468a0..d58cf07 100644 --- a/internal/broker/raise_live_test.go +++ b/internal/broker/raise_live_test.go @@ -28,15 +28,12 @@ func aLiveBus(t *testing.T) *JetStream { t.Fatal(err) } t.Cleanup(js.Close) - // Deleted before, so what this test asserts is what it finds — and after, so the next test does - // not inherit it. Deleting a stream takes its consumers with it, which is why this is enough. - clear := func() { - for _, s := range MeshStreams() { - _ = js.Context().DeleteStream(s.Name) - } - } - clear() - t.Cleanup(clear) + // **Nothing is deleted here, deliberately.** These objects are the mesh's own and every live + // test in every package shares one server: a test that deleted a stream to get a clean slate + // took it out from under whatever was running beside it, and the failure landed in the other + // test as "stream not found" — which reads as a bug in the code under test. Raise is idempotent + // by requirement, so asserting against whatever is already there is both safe and the realistic + // case. return js } diff --git a/internal/link/enrol_bus_credential_test.go b/internal/link/enrol_bus_credential_test.go new file mode 100644 index 0000000..9a35404 --- /dev/null +++ b/internal/link/enrol_bus_credential_test.go @@ -0,0 +1,107 @@ +package link_test + +import ( + "crypto/ed25519" + "crypto/rand" + "testing" + "time" + + "golang.org/x/crypto/bcrypt" + + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// What a node is given to come back with, on the bus being built. +// +// **The credential becomes usable at the next composition, not when it is made**, which is the one +// real difference from the bus the mesh runs on today: there a management call makes it live at once. +// So what has to be true here is that the mesh recorded it and told the node, and the rest is a push. + +// aMeshReadyToEnrol is both stores with a signing key established, which a control plane does at +// start: one that cannot sign is one whose declarations every node correctly refuses. +func aMeshReadyToEnrol(t *testing.T) (*inventory.Inventory, *identity.Identity) { + t.Helper() + inv := inventory.ForTest(t) + ident := identity.ForTest(t) + if _, err := ident.Establish(t.Context()); err != nil { + t.Fatal(err) + } + return inv, ident +} + +func aTokenFor(t *testing.T, inv *inventory.Inventory, node string) (string, ed25519.PublicKey) { + t.Helper() + ctx := t.Context() + if _, err := inv.AddNode(ctx, node); err != nil { + t.Fatal(err) + } + issued, err := inv.IssueToken(ctx, node, time.Hour) + if err != nil { + t.Fatal(err) + } + public, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return issued.Secret, public +} + +// A node enrolling onto the bus being built is told a password of its own, and the mesh keeps only +// its hash — which is what the next composition writes into the bus's user list. +func TestANodeEnrollingOnTheNewBusIsMintedACredentialTheMeshOnlyHashes(t *testing.T) { + inv, ident := aMeshReadyToEnrol(t) + ctx := t.Context() + secret, public := aTokenFor(t, inv, "anchor") + + reply, err := link.Enrolment{Inventory: inv, Identity: ident, OnNATS: true}.Enrol(ctx, link.EnrolRequest{ + Node: "anchor", Secret: secret, PublicKey: public}) + if err != nil { + t.Fatal(err) + } + if reply.Password == "" { + t.Fatal("the node was told no password, so it keeps a one-time secret as a credential") + } + if reply.Password == secret { + t.Fatal("the node was handed the token's own secret back: a credential that lives for years " + + "must not be the string that was pasted into a terminal") + } + + // Recorded under the name the composed file will use, and as a hash: a credential recoverable + // from the mesh's store is one whose blast radius is the store's. + hash, known, err := inv.BusUserHash(ctx, "node.anchor") + if err != nil || !known { + t.Fatalf("the mesh kept no credential for the node it enrolled: %v %v", known, err) + } + if hash == reply.Password { + t.Fatal("the store holds the password itself") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(reply.Password)); err != nil { + t.Fatalf("what the mesh kept does not verify what it told the node: %v", err) + } +} + +// On the bus the mesh runs on today, with no management configured, nothing is minted and the node is +// told so by being given no password — it keeps the token's secret, which it says out loud. +// +// **This is the check that the switch is a switch.** A node enrolling on one bus must not come away +// with a credential for the other: it would be half-moved, and nothing anywhere would say which half. +func TestANodeEnrollingOnTheOldBusIsMintedNoCredentialForTheNewOne(t *testing.T) { + inv, ident := aMeshReadyToEnrol(t) + ctx := t.Context() + secret, public := aTokenFor(t, inv, "anchor") + + reply, err := link.Enrolment{Inventory: inv, Identity: ident}.Enrol(ctx, link.EnrolRequest{ + Node: "anchor", Secret: secret, PublicKey: public}) + if err != nil { + t.Fatal(err) + } + if reply.Password != "" { + t.Fatalf("a node on the old bus was given a password from nowhere: %q", reply.Password) + } + if _, known, err := inv.BusUserHash(ctx, "node.anchor"); err != nil || known { + t.Fatalf("a node enrolling on the old bus was given a credential for the new one: %v %v", + known, err) + } +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 657ec21..75187d5 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -28,6 +28,15 @@ type Enrolment struct { Identity *identity.Identity Management *broker.Management Broker broker.Broker + + // OnNATS says the mesh's own traffic is on the bus being built, so a node's credential is + // minted into the mesh's records and composed into the bus's user list rather than pushed + // through a management call (novox/hq design 25 §4). + // + // **One bus, and a node gets a credential for exactly one** — refused at start if the + // controller is told about both (broker.MustBeOneBus), because a node holding a credential for + // each is one that could be half-moved, and nothing would say which half. + OnNATS bool } // Enrol records what the node presented and spends the token. @@ -159,7 +168,33 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol // the spend and not before: a replaced password on an attempt that failed would be held by // nobody. If the broker will not take it now, the enrolment still stands — the node keeps // the token's secret as its password, which it is told, and which is said here. - if e.Management != nil { + switch { + case e.OnNATS: + // **Minted into the mesh's records, not pushed to a server.** The bus's users are a file + // the controller composes, so a credential becomes usable at the next composition rather + // than at the moment it is made — and the plaintext is returned once, here, and then exists + // only on the machine it was sealed to. + // + // The node reconnects as itself and may be refused until that composition reaches the + // machine running the bus. That is what the host's reconnect backoff is for and it is + // survivable by design (ADR 0004: disconnection is an ordinary situation); waiting for the + // push here would hold an enrolment open for as long as a declaration takes to apply. + password, err := e.Inventory.MintBusPassword(ctx, inventory.BusUser{ + Username: broker.Principal{Kind: broker.KindNode, Node: node.Name}.Username(), + Kind: inventory.BusNode, + Node: node.Name, + }) + if err != nil { + // Not fatal to the enrolment: the node is recorded and the token is spent, and a node + // that keeps the token's secret is told so. Said loudly, because until this is minted + // the machine has no credential of its own. + log.Printf("%s is enrolled and the mesh could not mint its bus credential, so it keeps "+ + "the token's secret as its password: %v", node.Name, err) + } else { + reply.Password = password + } + + case e.Management != nil: password, err := freshPassword() if err != nil { return EnrolReply{}, err diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go index 2f4a58e..0dc2edf 100644 --- a/internal/link/receive_nats_test.go +++ b/internal/link/receive_nats_test.go @@ -36,19 +36,30 @@ func aBus(t *testing.T) *broker.JetStream { } t.Cleanup(js.Close) - // The mesh's own streams and consumers, asserted the way the controller asserts them — and - // torn down after, so one test's held message is never another's surprise. - for _, s := range broker.MeshStreams() { - _ = js.Context().DeleteStream(s.Name) - } + // **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong. + // + // The streams are emptied rather than deleted and recreated, because delete-then-add is not a + // reset: the server's teardown races the creation, and a test then inherits the previous one's + // messages — which reads as a redelivery bug in the code under test. + // + // The consumers are removed, because deleting a stream used to take them with it and purging + // does not. A durable *push* consumer that survives between tests keeps pushing to a delivery + // subject the previous test's subscription has gone from: the messages count as delivered, go + // nowhere, and the next test waits out its timeout for an announcement the server believes it + // already sent. The controller recreates what it needs on start, so leaving none is correct. if err := broker.AssertMeshStreams(js); err != nil { t.Fatal(err) } - t.Cleanup(func() { - for _, s := range broker.MeshStreams() { - _ = js.Context().DeleteStream(s.Name) + clean := func() { + for _, c := range broker.MeshConsumers() { + _ = js.Context().DeleteConsumer(c.Stream, c.Name) } - }) + for _, s := range broker.MeshStreams() { + _ = js.Context().PurgeStream(s.Name) + } + } + clean() + t.Cleanup(clean) return js } -- 2.54.0 From d65c37caadca94a6ebc83b9a8e136c930fb982a4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 14:13:52 +0200 Subject: [PATCH 29/39] The controller could not answer an enrolment, and a probe on an open server said it could MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while reasoning about issue 127's replay question, in code committed earlier today. The controller's permissions granted no inbox at all, so the answer to every enrolment on the mesh would have been refused — "Permissions Violation for Publish to _INBOX.enrol.anchor…" — while the controller logged that it had enrolled the node. **`allow_responses` does not cover it, and that is the trap.** It permits one reply to the reply subject of a message the user received, and a message a JetStream consumer delivers has had that field claimed for the consumer's own ack address (design 25 §2). The address the controller actually answers is the one the request carried in its *payload*, which the server does not recognise as a reply subject at all. The two mechanisms look interchangeable and are not. **My earlier verification could not have caught this.** The live enrolment tests run against a server with no accounts and no permissions, so they exercise the subjects and the round trip and nothing about authority. Composing the real configuration and running a server on it is what found it. Granted the enrolment inbox space and nothing wider: nothing but an enrolling node ever subscribes under that prefix, each scoped to its own token's, so the controller publishing there is the mesh answering enrolments and reaches nothing else. Confirmed against the permissioned server both ways — the answer arrives, and a node's own inbox is still refused. Pinned as a rule that needs no server: whatever an enrolling node subscribes, the controller must be able to publish to, and a node's, a module's and a person's inbox must stay out of reach. That check is a subject-pattern match rather than a string compare, so a grant that widened by a wildcard would not slip past it. It also bears on 127's open question about who replays a build announcement: an answer to a *module's* inbox would need `_INBOX.>`, which is exactly the blanket grant design 25 §4 refuses. So the catch-up cannot become an inbox reply. --- internal/broker/nats.go | 22 ++++++++- internal/broker/nats_test.go | 68 ++++++++++++++++++++++++++ internal/broker/testdata/composed.conf | 2 +- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 159fbca..579079e 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -76,6 +76,10 @@ type Principal struct { PasswordHash string } +// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the +// controller may answer an enrolment is derived from the same constant the user is named from. +const enrolmentPrefix = "enrol" + // safeSubject refuses anything that would change the meaning of a subject rather than sit inside // one. A name carrying a dot would silently widen a permission by adding a token; a name carrying // `>` or `*` would widen it to a wildcard, which is the whole authority model gone. @@ -104,7 +108,7 @@ func (p Principal) Username() string { // the mesh holds one live claim per record, and the node's name is the one identifier both // sides already have before anything else is agreed. It is also exactly what the other // transport does, where the account is named after the node and the secret is its password. - return "enrol." + p.Node + return enrolmentPrefix + "." + p.Node } return "" } @@ -163,6 +167,22 @@ func PermissionsFor(p Principal) (Permissions, error) { sub = append(sub, ControllerFollows...) pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>") + // **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That + // permits one reply to the reply subject of a message the user received — and a message a + // JetStream consumer delivers has had that field claimed for the consumer's own ack address + // (design 25 §2), so the address the controller actually answers is the one the request + // carried in its payload, which is not a reply subject as the server understands it. + // + // Verified against a real server before this line existed: the answer was refused with + // "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the + // mesh would have timed out while the controller logged success. + // + // **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and + // this is not it: nothing but an enrolling node ever subscribes under this prefix, each + // scoped to its own token's, so the controller publishing here is the mesh answering + // enrolments and can reach nothing else. + pub = append(pub, "_INBOX."+enrolmentPrefix+".>") + case KindPerson: // Tools, and nothing else. Every subject a person may publish is a tool call; a person // who could publish an event would be able to claim a module said something. diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index f5c2500..6ba56b1 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -256,3 +256,71 @@ func TestAToolGrantThatNamesNoToolIsRefused(t *testing.T) { t.Fatal("a grant naming a module but no tool was accepted") } } + +// The controller can answer an enrolment, and reach no other inbox. +// +// **`allow_responses` does not cover this and that is the trap.** It permits one reply to the reply +// subject of a message the user received — and a message a JetStream consumer delivers has had that +// field claimed for the consumer's own ack address, so the address the controller actually answers is +// the one the request carried in its payload, which the server does not recognise as a reply subject +// at all. +// +// Found against a real server, after a live test on an *unpermissioned* one had passed: every +// enrolment on the mesh would have timed out while the controller logged success. +func TestTheControllerCanAnswerAnEnrolmentAndReachNoOtherInbox(t *testing.T) { + ctl, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + enrolling, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + + // Whatever the enrolling node waits on, the controller must be able to publish to. + if len(enrolling.Subscribe) != 1 { + t.Fatalf("an enrolling node subscribes %v, and this test knows only how to check one", + enrolling.Subscribe) + } + waitsOn := enrolling.Subscribe[0] + if !covers(ctl.Publish, waitsOn) { + t.Fatalf("the controller may publish %v, none of which reaches %s — so every enrolment on "+ + "the mesh times out while the controller logs success", ctl.Publish, waitsOn) + } + + // And nothing wider. A node's own inbox and a module's are not the controller's to write into: + // that is the blanket grant design 25 §4 refuses. + for _, other := range []string{"_INBOX.node.anchor.x", "_INBOX.one.shop.x", "_INBOX.person.ada.x"} { + if covers(ctl.Publish, other) { + t.Errorf("the controller can publish to %s, which is an inbox privacy the permission "+ + "list is the only thing protecting", other) + } + } +} + +// covers says whether any granted subject pattern admits one concrete subject, with NATS's own +// wildcard meanings: `*` is one token, `>` is the rest. +func covers(granted []string, subject string) bool { + want := strings.Split(subject, ".") + for _, pattern := range granted { + if admits(strings.Split(pattern, "."), want) { + return true + } + } + return false +} + +func admits(pattern, subject []string) bool { + for i, token := range pattern { + if token == ">" { + return i < len(subject) + } + if i >= len(subject) { + return false + } + if token != "*" && token != subject[i] { + return false + } + } + return len(pattern) == len(subject) +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index dd7da95..6bcd013 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -23,7 +23,7 @@ accounts { MESH { users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "mesh.build.>", "mesh.control.>", "mesh.node.>"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.build.>", "mesh.control.>", "mesh.node.>"] } subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.build.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.module.mesh-catalog.catching-up", "mesh.mod.mesh-catalog.event.module.mesh-catalog.upgraded"] } allow_responses: { max: 1, ttl: "1m" } } } -- 2.54.0 From 05ff6065d0301a4f0606f4da0be1f228ec43c895 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 14:43:16 +0200 Subject: [PATCH 30/39] Event names are checked now, per manifest and across the catalogue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue 127 stood because nothing compared the two halves. Every manifest was well-formed on its own and every derivation correct on its own, and no cross-module subscription in the mesh matched anything — a subscription that matches nothing is not an error, it is silence. Two checks, because the mistake is possible at two scales. Per manifest: an event is a local name, and `module.` is refused with the name to write instead. A module emitting under what reads as another module's name is refused too, pointing at the seat, where a name outlives whoever holds it. Across the catalogue: where a consumed event's emitter is present, it must emit that event. It cannot demand a live emitter for everything — a module lives in its own repository and may be installed long before the one whose events it wants — so the rule is narrower and still catches this. It found two real dangling subscriptions the moment it ran. Wildcards were undecided and two manifests needed them: `*` is one name and `**` is the rest, spelled the mesh's way and derived to `>` here and `#` on the old bus. A manifest naming either would stop being true when the wire changed, which is the whole reason names are local. And the field documentation taught the old form, examples included — which is why the drift was uniform across 37 manifests rather than scattered. Nobody was guessing; everybody followed the comment. --- internal/broker/agreement.go | 125 +++++++++++++++ internal/broker/agreement_catalogue_test.go | 108 +++++++++++++ internal/broker/derived.go | 8 +- internal/broker/nats.go | 56 ++++++- internal/catalogue/events.go | 161 ++++++++++++++++++++ internal/catalogue/manifest.go | 32 +++- internal/catalogue/no_subjects_test.go | 48 ++++++ internal/inventory/busrecords.go | 4 +- 8 files changed, 526 insertions(+), 16 deletions(-) create mode 100644 internal/broker/agreement.go create mode 100644 internal/broker/agreement_catalogue_test.go create mode 100644 internal/catalogue/events.go diff --git a/internal/broker/agreement.go b/internal/broker/agreement.go new file mode 100644 index 0000000..3f4fc6b --- /dev/null +++ b/internal/broker/agreement.go @@ -0,0 +1,125 @@ +package broker + +import ( + "fmt" + "sort" + "strings" +) + +// Do the emitters and the consumers of a catalogue agree? +// +// **The check that was missing** (novox/hq 04-ISSUES/127). Every manifest was individually +// well-formed and every derivation individually correct, and no cross-module subscription in the +// mesh matched anything: a consumer's declaration derived into a namespace nobody publishes to. +// Nothing failed, because a subscription that matches nothing is not an error — it is silence. +// +// The comparison has to be over the whole catalogue, because the two halves live in different +// manifests, and it cannot simply demand that every consumed event have a live emitter: a module +// may be installed long before the one whose events it wants. So the rule is narrower and still +// catches this: **where the emitter is present, it must emit what the consumer asked for.** + +// AConsumer is one module's interest in another's events, as this check needs it. +type AConsumer struct { + Module string + Consumes []string +} + +// AnEmitter is one module's events. +type AnEmitter struct { + Module string + Emits []string +} + +// Disagreements are the consumed events whose emitter is in the catalogue and does not emit them. +// +// Returned as sentences rather than as structs: every one of them is read by a person deciding +// whether a manifest or a catalogue is wrong, and a pair of names without the reason is a puzzle. +func Disagreements(emitters []AnEmitter, consumers []AConsumer, seats []DeclaredSeat) []string { + emits := map[string]map[string]bool{} + for _, e := range emitters { + if emits[e.Module] == nil { + emits[e.Module] = map[string]bool{} + } + for _, name := range e.Emits { + emits[e.Module][name] = true + } + } + // A seat's events are published by its holder under the seat's name, so a consumer naming the + // seat is naming something real even though no module declares it as its own. + for _, s := range seats { + if len(s.Emits) == 0 { + continue + } + if emits[s.Name] == nil { + emits[s.Name] = map[string]bool{} + } + for _, name := range s.Emits { + emits[s.Name][name] = true + } + } + + var out []string + for _, c := range consumers { + for _, pattern := range c.Consumes { + emitter, event, named := strings.Cut(pattern, ".") + // Every event from everyone, or every event from one module: both are deliberate and + // neither names a particular event to check. + if !named || emitter == "*" || emitter == catalogueTheRest || event == catalogueTheRest { + continue + } + known, present := emits[emitter] + if !present { + // Not installed here, which is ordinary: a module lives in its own repository and + // may be registered later. Nothing to compare, so nothing to say. + continue + } + if matchesAny(event, known) { + continue + } + out = append(out, fmt.Sprintf( + "%s consumes %q and %s emits %s — so that subscription would match nothing, and "+ + "nothing would report it", + c.Module, pattern, emitter, listOf(known))) + } + } + sort.Strings(out) + return out +} + +// matchesAny says whether one of an emitter's event names satisfies a consumer's pattern. +func matchesAny(pattern string, emitted map[string]bool) bool { + want := strings.Split(pattern, ".") + for name := range emitted { + if matches(want, strings.Split(name, ".")) { + return true + } + } + return false +} + +func matches(pattern, name []string) bool { + for i, part := range pattern { + if part == catalogueTheRest { + return i < len(name) + } + if i >= len(name) { + return false + } + if part != "*" && part != name[i] { + return false + } + } + return len(pattern) == len(name) +} + +func listOf(names map[string]bool) string { + if len(names) == 0 { + return "nothing" + } + out := make([]string, 0, len(names)) + for n := range names { + out = append(out, n) + } + sort.Strings(out) + return strings.Join(out, ", ") +} diff --git a/internal/broker/agreement_catalogue_test.go b/internal/broker/agreement_catalogue_test.go new file mode 100644 index 0000000..9c4a348 --- /dev/null +++ b/internal/broker/agreement_catalogue_test.go @@ -0,0 +1,108 @@ +package broker + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// **Do the catalogue's emitters and consumers agree?** +// +// This is the check whose absence let issue 127 stand: every manifest was individually well-formed, +// every derivation individually correct, and no cross-module subscription in the mesh matched +// anything. A subscription that matches nothing is not an error — it is silence — so nothing +// anywhere reported it. +// +// It compares what one manifest asks to hear against what another says it emits. It cannot demand +// that every consumed event have a live emitter, because a module lives in its own repository and +// may be registered long before the one whose events it wants. Where the emitter *is* here, it must +// emit what the consumer asked for. +func TestTheCataloguesEmittersAndConsumersAgree(t *testing.T) { + emitters, consumers, seats := theCataloguesEvents(t) + + if bad := Disagreements(emitters, consumers, seats); len(bad) > 0 { + t.Fatalf("%d subscription(s) in the catalogue would match nothing:\n %s", + len(bad), strings.Join(bad, "\n ")) + } +} + +// And the check itself catches the thing it exists for, so it cannot pass by doing nothing. +func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) { + bad := Disagreements( + []AnEmitter{{Module: "builder", Emits: []string{"built"}}}, + []AConsumer{{Module: "mesh-catalog", Consumes: []string{"builder.finished"}}}, + nil) + if len(bad) != 1 { + t.Fatalf("a consumer asking for an event its emitter does not emit was not caught: %v", bad) + } + if !strings.Contains(bad[0], "builder.finished") || !strings.Contains(bad[0], "built") { + t.Fatalf("the report names neither what was asked for nor what is emitted: %s", bad[0]) + } + + // A module that is not here is not a disagreement: it may be registered later. + if bad := Disagreements(nil, + []AConsumer{{Module: "plex", Consumes: []string{"sonarr.download.completed"}}}, nil); len(bad) != 0 { + t.Fatalf("a consumer whose emitter is not installed was reported: %v", bad) + } + + // A wildcard over emitters is deliberate and names no particular event to check. + if bad := Disagreements([]AnEmitter{{Module: "sonarr", Emits: []string{"download.completed"}}}, + []AConsumer{{Module: "plex", Consumes: []string{"*.download.completed"}}}, nil); len(bad) != 0 { + t.Fatalf("a wildcard over emitters was reported: %v", bad) + } + + // An event published under a seat's name is real even though no module declares it as its own. + if bad := Disagreements(nil, + []AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}}, + []DeclaredSeat{{Name: "mesh-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 { + t.Fatalf("an event a seat emits was reported as matching nothing: %v", bad) + } +} + +func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) { + t.Helper() + root := filepath.Join("..", "..", "..", "mesh-catalog", "modules") + entries, err := os.ReadDir(root) + if err != nil { + t.Skipf("catalogue sibling not present: %v", err) + } + var emitters []AnEmitter + var consumers []AConsumer + var seats []DeclaredSeat + for _, e := range entries { + if !e.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json")) + if err != nil { + continue + } + var m struct { + Module string `json:"module"` + Emits []string `json:"emits"` + Consumes []string `json:"consumes"` + Seats []struct { + Name string `json:"name"` + Emits []string `json:"emits"` + } `json:"seats"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", e.Name(), err) + } + if len(m.Emits) > 0 { + emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits}) + } + if len(m.Consumes) > 0 { + consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes}) + } + for _, s := range m.Seats { + seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits}) + } + } + if len(emitters) == 0 { + t.Skip("no manifests found beside this checkout") + } + return emitters, consumers, seats +} diff --git a/internal/broker/derived.go b/internal/broker/derived.go index 97d3174..b5e15c0 100644 --- a/internal/broker/derived.go +++ b/internal/broker/derived.go @@ -85,8 +85,12 @@ func SeatStreams(seats []DeclaredSeat) []Stream { // package stays free of the catalogue's own types — the same reason the host mirrors the // contracts instead of importing the sdk. type DeclaredSeat struct { - Name string - Accepts []string + Name string + Accepts []string + // Emits are the verbs the seat's holder publishes under the seat's own name. An event about a + // role belongs here rather than in the holder's namespace, because the name then outlives + // whoever fills it (novox/hq 04-ISSUES/127). + Emits []string RetainSeconds int } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 579079e..4e0a7aa 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -241,12 +241,11 @@ func PermissionsFor(p Principal) (Permissions, error) { // 2. What it consumes, by the emitter's own subject — an event is addressed to its // emitter, because the emitter's identity is the meaning (ADR 0118). for _, c := range p.Consumes { - emitter, event, ok := strings.Cut(c, ".") - if !ok { - return Permissions{}, fmt.Errorf( - "%q does not name an emitter and an event: a consumed event is .", c) + subject, err := consumedSubject(c) + if err != nil { + return Permissions{}, err } - sub = append(sub, "mesh.mod."+emitter+".event."+event) + sub = append(sub, subject) } // 3. Seats it holds: full participation. @@ -331,6 +330,53 @@ func seatSubject(s Seat, kind, verb string) string { // // They are derived here, beside the permission that must match them, because two places deriving // the same name is how a module ends up unable to ack its own deliveries. +// consumedSubject is where a consumed event lands, from the local pattern a module declared. +// +// **The mesh's wildcards become this transport's** (design 29 §1): `*` is one name on both, and `**` +// — the rest — is `>` here. A module writes neither transport's spelling, so a manifest stays correct +// when the wire changes, which is the whole reason names are local. +// +// `**` on its own is every event from every module: the emitter is any, the event is anything. An +// audit logger wants exactly that and says so in one token. +func consumedSubject(pattern string) (string, error) { + if pattern == catalogueTheRest { + return "mesh.mod.*.event.>", nil + } + emitter, event, named := strings.Cut(pattern, ".") + if !named || emitter == "" || event == "" { + return "", fmt.Errorf( + "%q does not name an emitter and an event: a consumed event is ., or "+ + "%q for every event", pattern, catalogueTheRest) + } + if emitter == catalogueTheRest { + return "", fmt.Errorf("%q stands for the rest of a name, so it cannot name the emitter", catalogueTheRest) + } + // Each name is checked before it becomes a subject: a name carrying a dot would add a token and + // silently widen the permission, which is the whole reason safeSubject exists. + var out []string + for _, part := range strings.Split(event, ".") { + switch part { + case catalogueTheRest: + out = append(out, ">") + case "*": + out = append(out, "*") + default: + if !safeSubject.MatchString(part) { + return "", fmt.Errorf("%q cannot be part of a subject: it would widen the permission", part) + } + out = append(out, part) + } + } + if emitter != "*" && !safeSubject.MatchString(emitter) { + return "", fmt.Errorf("%q cannot name an emitter: it would widen the permission", emitter) + } + return "mesh.mod." + emitter + ".event." + strings.Join(out, "."), nil +} + +// catalogueTheRest is the mesh's wildcard for "the rest of a name", duplicated from the catalogue +// package for the one direction of dependency the build queue's name is duplicated for. +const catalogueTheRest = "**" + func consumerStream(p Principal) string { switch p.Kind { case KindModule: diff --git a/internal/catalogue/events.go b/internal/catalogue/events.go new file mode 100644 index 0000000..90dda63 --- /dev/null +++ b/internal/catalogue/events.go @@ -0,0 +1,161 @@ +package catalogue + +import ( + "fmt" + "regexp" + "strings" +) + +// What a module may call an event, and what a consumer may ask for. +// +// A module names an event **locally**: `order.placed`, not a subject and not a routing key +// (design 29 §1). A consumer names the emitter and the event: `billing.order.placed`. The mesh +// derives the subject from those, so reorganising the subject space leaves every manifest correct. +// +// **Nothing checked this until every manifest in the catalogue was wrong the same way** +// (novox/hq 04-ISSUES/127). All thirty-seven kept the old bus's routing key — +// `module..` — which the derivation read as "a module called `module`", so every +// cross-module subscription in the mesh pointed at a namespace nobody publishes to. Nothing failed: +// the services started and none of them reacted. The documentation on these fields taught the old +// form too, which is why the drift was uniform rather than scattered. + +// eventName is one name in a local event: lower-case, and no wildcard. +var eventName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// The wildcards a consumer may use, spelled the mesh's way and derived to whatever the transport +// spells them as. +// +// **A manifest holds no transport token**, which is the whole point of naming locally: the bus the +// mesh runs on today spells these `*` and `#`, and the one being built spells them `*` and `>`. A +// manifest that said either would be a manifest that stopped being true when the wire changed. +const ( + // OneName stands for exactly one name. + OneName = "*" + // TheRest stands for one or more names, and may only come last. + TheRest = "**" +) + +// EventProblems is what is wrong with a manifest's events. +// +// Refused at registration, because the alternative is a module that installs, starts, connects and +// reacts to nothing — and every log line says it is fine. +func EventProblems(m Manifest) []string { + var problems []string + + for _, e := range m.Emits { + if was, stale := staleEventForm(e, m.Module); stale { + problems = append(problems, fmt.Sprintf( + "%s emits %q, which is the old bus's routing key. An event is named locally now, so "+ + "write %q — the mesh derives the subject (novox/hq design 29 §1)", + m.Module, was, strings.TrimPrefix(was, "module."+m.Module+"."))) + continue + } + if strings.HasPrefix(e, "module.") { + problems = append(problems, fmt.Sprintf( + "%s emits %q: `module.` is reserved, because it is how the old bus spelled a "+ + "routing key and an event named that way derives into a namespace nobody owns", + m.Module, e)) + continue + } + if err := localName(e); err != nil { + problems = append(problems, fmt.Sprintf("%s emits %q: %v", m.Module, e, err)) + continue + } + // **Its own name, never another's.** The bus enforces that a namespace belongs to the module + // it is named for, so an event named for somebody else cannot be published at all. If the + // event is about a role rather than about this module, it belongs on the seat: a name that + // is stable across whoever fills it (04-ISSUES/127). + if first, _, split := strings.Cut(e, "."); split && isAModuleNameOtherThan(first, m.Module) { + problems = append(problems, fmt.Sprintf( + "%s emits %q, which reads as another module's event. A module publishes under its "+ + "own name only. If this is about a role rather than about %s, declare it on that "+ + "seat, where the name survives the holder changing", + m.Module, e, m.Module)) + } + } + + for _, c := range m.Consumes { + if strings.HasPrefix(c, "module.") { + problems = append(problems, fmt.Sprintf( + "%s consumes %q, which is the old bus's pattern. A consumed event names its emitter "+ + "and the event: write %q", m.Module, c, strings.TrimPrefix(c, "module."))) + continue + } + if c == "#" { + problems = append(problems, fmt.Sprintf( + "%s consumes %q, which is the old bus's wildcard for everything. Write %q", + m.Module, c, TheRest)) + continue + } + if err := consumePattern(c); err != nil { + problems = append(problems, fmt.Sprintf("%s consumes %q: %v", m.Module, c, err)) + } + } + return problems +} + +// staleEventForm says an emitted name is this module's own old routing key, and what it was. +func staleEventForm(event, module string) (string, bool) { + return event, module != "" && strings.HasPrefix(event, "module."+module+".") +} + +// isAModuleNameOtherThan says a first token names some module of this mesh that is not this one. +// +// Only the mesh's own seats and the catalogue could answer this properly, and neither is reachable +// from a parser given one manifest. So this catches the case that actually happened — a name that +// is a *provision* the mesh defines, which is where "another module's event" comes from in practice +// — and the whole-catalogue check catches the rest. +func isAModuleNameOtherThan(first, module string) bool { + if first == module || first == "" { + return false + } + if _, isASeat := SeatNamed(first); isASeat { + return true + } + if _, isASeat := SeatDelivering(first); isASeat { + return true + } + return false +} + +// localName checks one event name: dot-separated names, no wildcards, nothing else. +func localName(event string) error { + if event == "" { + return fmt.Errorf("an event needs a name") + } + for _, part := range strings.Split(event, ".") { + if part == OneName || part == TheRest { + return fmt.Errorf("an emitted event names one event, so it carries no wildcard") + } + if !eventName.MatchString(part) { + return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part) + } + } + return nil +} + +// consumePattern checks a consumed pattern: the emitter, then the event, with wildcards. +func consumePattern(pattern string) error { + if pattern == "" { + return fmt.Errorf("a consumed event needs an emitter and an event") + } + parts := strings.Split(pattern, ".") + for i, part := range parts { + switch { + case part == TheRest: + if i != len(parts)-1 { + return fmt.Errorf("%q stands for the rest of a name, so nothing may follow it", TheRest) + } + case part == OneName: + case !eventName.MatchString(part): + return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part) + } + } + // `**` alone is every event from every module, which the audit logger wants and says plainly. + if len(parts) == 1 && parts[0] != TheRest { + return fmt.Errorf( + "%q names an emitter and no event. Write ., or %q for every event", + pattern, TheRest) + } + return nil +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index b874536..ed3b37f 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -176,15 +176,29 @@ type Manifest struct { // Requires are names that must be provided by something assigned to the same node. Requires []string `json:"requires,omitempty"` - // Emits are the event types this module publishes onto the broker — dotted topic keys, e.g. - // "module.umami.site.created". Declared so the mesh knows the event graph; events are - // provisioning's lighter sibling — 1:many and broadcast, no credential (novox/hq ADR 0041). + // Emits are the events this module publishes, named **locally**: `order.placed`, not a subject + // and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the + // subject space leaves this manifest correct. Events are provisioning's lighter sibling — 1:many + // and broadcast, no credential (novox/hq ADR 0041). + // + // A module publishes under its own name only. If the event is about a *role* rather than about + // this module, it belongs on that seat, where the name outlives whoever holds it. + // + // **This said "dotted topic keys, e.g. module.umami.site.created" until 04-ISSUES/127**, which + // is the old bus's routing key, and is why every manifest in the catalogue had the same mistake: + // nobody was guessing, everybody followed this comment. Emits []string `json:"emits,omitempty"` - // Consumes are the event patterns this module subscribes to — topic patterns over module, - // mesh and node events alike, e.g. "node.*.joined" or "#" (the audit logger). The runtime - // wires the subscription; the module ships the handler. A Consumes for an event nothing on - // the mesh Emits is a dangling edge. + // Consumes are the events this module reacts to, each naming its emitter and the event: + // `billing.order.placed`. `*` stands for one name and `**` for the rest, so `*.download.completed` + // is that event from any module and `**` is every event in the mesh. + // + // Spelled the mesh's way rather than the wire's, for the reason Emits is: the bus the mesh runs + // on today spells these `*` and `#`, the one being built spells them `*` and `>`, and a manifest + // naming either would stop being true when the wire changed. + // + // The runtime wires the subscription; the module ships the handler. A Consumes for an event + // nothing on the mesh Emits is a dangling edge. Consumes []string `json:"consumes,omitempty"` // Claims are singular resources. Two modules claiming one thing within a scope cannot both @@ -999,6 +1013,10 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, fmt.Sprintf("%s requires itself", m.Module)) } } + // What it may call an event, and what it may ask to hear (events.go). Checked here because a + // module whose event names are wrong installs, starts, connects and reacts to nothing, with + // every log line saying it is fine (novox/hq 04-ISSUES/127). + problems = append(problems, EventProblems(m)...) wellFormed := true for _, c := range m.Claims { if !name.MatchString(c.Name) { diff --git a/internal/catalogue/no_subjects_test.go b/internal/catalogue/no_subjects_test.go index f86a8d2..cb4b301 100644 --- a/internal/catalogue/no_subjects_test.go +++ b/internal/catalogue/no_subjects_test.go @@ -71,3 +71,51 @@ func TestNoManifestContainsASubject(t *testing.T) { } t.Logf("%d manifests hold no subject", checked) } + +// **Every module's event names are what design 29 says, across the whole catalogue.** +// +// The rule above holds by construction and turned out to be weaker than it reads: a manifest holds +// no subject, and every manifest in the catalogue still held the old bus's routing key, which +// derives into a namespace nobody owns (novox/hq 04-ISSUES/127). Nothing failed — the services +// started and none of them reacted. This is the check that was missing. +func TestEveryManifestsEventNamesAreLocal(t *testing.T) { + manifests := theCatalogue(t) + + var problems []string + for _, m := range manifests { + problems = append(problems, EventProblems(m)...) + } + if len(problems) > 0 { + t.Fatalf("the catalogue holds %d event name(s) the mesh would derive wrongly:\n %s", + len(problems), strings.Join(problems, "\n ")) + } +} + +// theCatalogue is every manifest beside this checkout, parsed the way registration parses one. +func theCatalogue(t *testing.T) []Manifest { + t.Helper() + root := filepath.Join("..", "..", "..", "mesh-catalog", "modules") + entries, err := os.ReadDir(root) + if err != nil { + t.Skipf("catalogue sibling not present: %v", err) + } + var out []Manifest + for _, e := range entries { + if !e.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json")) + if err != nil { + continue + } + var m Manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", e.Name(), err) + } + out = append(out, m) + } + if len(out) == 0 { + t.Skip("no manifests found beside this checkout") + } + return out +} diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index 61051f2..a9a6ae1 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -88,8 +88,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio Serves: m.Tools, } for _, c := range m.Claims { - // A seat the mesh defines for itself declares no protocol, so holding one grants nothing - // here — which is right: those seats say who does a job, not who may say what. + // A seat the mesh defines for itself carries no protocol, so holding one grants nothing here: + // those seats say who does a job, not who may say what. if s, declaredSomewhere := seats[c.Name]; declaredSomewhere { d.Holds = append(d.Holds, asSeat(s)) } -- 2.54.0 From 0c83ecf1b5cc2511152be030b9ae9693f723a59b Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 15:34:44 +0200 Subject: [PATCH 31/39] The mesh's own roles carry a protocol, and the build branch retires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0121, first half. The `mesh-*` seats said who does a job and nothing about what may be said to them or by them, so the mesh had roles it could not describe. They take the same three fields a module's seat has now, and the machinery that already derives a work queue, a holder's worker and a permission set from a declared seat does it for these too. The build-machine role accepts a build and emits an outcome, so `mesh.build.request`, `mesh.control.built` and the BUILDS stream are gone. A work queue shared by several build machines is what a seat's `accepts` already is, and keeping a second mechanism for it was two places a permission could be wrong. The controller's own side of a seat is a named list rather than something derived: it is not a module and declares no `uses`, so which roles the mesh itself submits work to has to be stated — and stating it makes that question answerable. Two things this caught: **The followed event subjects were hard-coded and had just gone stale.** They were written out while the catalogue still spelled its events as the old bus's routing keys, so converting those (issue 127) turned the pair into a controller listening to a subject nothing publishes — the same fault as the issue, from the other side. They derive from the emitter and the event name now, through the same function the permission uses, so the two cannot drift apart. **A role's queue exists before its holder**, checked against a real server, and asserting twice changes nothing. Work queues until somebody arrives to do it, so assigning a build machine later flushes the backlog instead of having lost it. --- cmd/mesh-controller/push.go | 6 ++++ internal/broker/derived.go | 6 ++-- internal/broker/nats.go | 17 ++++++++-- internal/broker/raise_live_test.go | 44 ++++++++++++++++++++++++++ internal/broker/streams.go | 33 +++++++++---------- internal/broker/streams_test.go | 1 - internal/broker/testdata/composed.conf | 4 +-- internal/catalogue/seats.go | 33 ++++++++++++++++++- internal/inventory/busrecords.go | 27 ++++++++++++++-- 9 files changed, 144 insertions(+), 27 deletions(-) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 1cafa57..f499281 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -713,6 +713,12 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) if err := broker.Raise(js, names); err != nil { return err } + // The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder, + // deliberately: work queues until somebody arrives to do it, so assigning a build machine a week + // after something started asking for builds flushes the backlog instead of having lost it. + if err := broker.RaiseSeats(js, inventory.MeshSeats(), nil); err != nil { + return err + } fmt.Printf("the bus at %s has its streams, and %d machine(s) can hear a declaration\n", address, len(names)) return nil diff --git a/internal/broker/derived.go b/internal/broker/derived.go index b5e15c0..b4b0cfa 100644 --- a/internal/broker/derived.go +++ b/internal/broker/derived.go @@ -89,8 +89,10 @@ type DeclaredSeat struct { Accepts []string // Emits are the verbs the seat's holder publishes under the seat's own name. An event about a // role belongs here rather than in the holder's namespace, because the name then outlives - // whoever fills it (novox/hq 04-ISSUES/127). - Emits []string + // whoever fills it (novox/hq ADR 0121, 04-ISSUES/127). + Emits []string + // Serves are the verbs the holder answers, request and reply. + Serves []string RetainSeconds int } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 4e0a7aa..45da6b0 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -76,6 +76,11 @@ type Principal struct { PasswordHash string } +// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than +// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a +// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable. +var meshSeatsTheControllerUses = []string{"mesh-build-machine"} + // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the // controller may answer an enrolment is derived from the same constant the user is named from. const enrolmentPrefix = "enrol" @@ -154,8 +159,16 @@ func PermissionsFor(p Principal) (Permissions, error) { case KindController: // The controller owns the mesh's own traffic and the streams. It is the only writer of // stream definitions (design 25 §3), so it alone reaches the JetStream API. - pub = []string{"mesh.control.>", "mesh.node.>", "mesh.build.>", "$JS.API.>"} - sub = []string{"mesh.control.>", "mesh.build.>", "$JS.API.>"} + pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"} + sub = []string{"mesh.control.>", "$JS.API.>"} + + // Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A + // build is the one today: the controller asks, and reads the answer from the seat's event + // like the catalogue does — which is why no holder needs to publish into anybody's inbox. + for _, seat := range meshSeatsTheControllerUses { + pub = append(pub, "mesh.seat."+seat+".accept.>") + sub = append(sub, "mesh.seat."+seat+".event.>") + } // The two events it reacts to, and its ack subject on the stream they arrive from // (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the diff --git a/internal/broker/raise_live_test.go b/internal/broker/raise_live_test.go index d58cf07..6281679 100644 --- a/internal/broker/raise_live_test.go +++ b/internal/broker/raise_live_test.go @@ -99,3 +99,47 @@ func TestTheControlConsumerDoesNotDeadLetterBeforeTheControllerGivesUp(t *testin "before the controller finished deciding about it", info.Config.MaxDeliver) } } + +// A role's work queue exists before anybody holds it, against a real server. +// +// **The queue before the holder is the point** (novox/hq ADR 0121): work queues until somebody arrives +// to do it, so assigning a build machine a week after something started asking for builds flushes the +// backlog instead of having lost it. A stream created at assignment would make "the holder is not here +// yet" mean "your requests are gone". +func TestRaisingAMeshRolesWorkQueue(t *testing.T) { + js := aLiveBus(t) + seats := []DeclaredSeat{{Name: "mesh-build-machine", Accepts: []string{"build"}, + Emits: []string{"built"}}} + t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") }) + + if err := RaiseSeats(js, seats, nil); err != nil { + t.Fatalf("a real server refused a role's work queue: %v", err) + } + info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE") + if err != nil { + t.Fatalf("the role has no work queue: %v", err) + } + if info.Config.Retention != nats.WorkQueuePolicy { + t.Errorf("the queue retains as %v: work a holder took must leave it, or the next holder does "+ + "it again", info.Config.Retention) + } + if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "mesh.seat.mesh-build-machine.accept.>" { + t.Errorf("it carries %v rather than the role's own inbound subjects", info.Config.Subjects) + } + // Nobody holds it, so there is no worker — and asserting again changes nothing, because this runs + // on every start. + if err := RaiseSeats(js, seats, nil); err != nil { + t.Fatalf("asserting a role's queue a second time failed, so a restart would: %v", err) + } + + // And once somebody holds it, the worker appears on that same queue. + if err := RaiseSeats(js, seats, map[string]Holder{ + "mesh-build-machine": {Node: "anchor", Module: "builder"}, + }); err != nil { + t.Fatal(err) + } + if _, err := js.Context().ConsumerInfo("SEAT_MESH_BUILD_MACHINE", + "SEAT_MESH_BUILD_MACHINE_worker"); err != nil { + t.Fatalf("the holder got no worker on the role's queue: %v", err) + } +} diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 7b0d11a..87748fb 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -63,8 +63,10 @@ type Stream struct { func MeshStreams() []Stream { return []Stream{ { - Name: "CONTROL", - Subjects: []string{"mesh.control.*.report", "mesh.control.enrol", "mesh.control.built"}, + Name: "CONTROL", + // A build's outcome is no longer here: it is the build-machine seat's own event, so one + // publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121). + Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, Retention: RetentionWorkQueue, Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " + "store is away and the message is redelivered; nothing is dropped", @@ -76,12 +78,6 @@ func MeshStreams() []Stream { Why: "one declaration per node, always the newest; a node that sees sequence n refuses " + "n-1 by construction (issue 107)", }, - { - Name: "BUILDS", - Subjects: []string{"mesh.build.request"}, - Retention: RetentionWorkQueue, - Why: "at least once, one builder at a time; a builder that dies mid-build has its message redelivered", - }, { Name: "EVENTS", // A seat's own events ride here too: they are 1:many like any event, and the @@ -167,15 +163,20 @@ const ControllerName = "controller" // ControllerFollows are the events the controller reacts to: the catalogue saying a module's // current version moved, and a catalogue that has just started saying it may have missed builds. // -// **These carry the local names the manifests hold today**, which still spell an event the way a -// routing key on the bus the mesh has does — `module..` rather than design 29's bare -// verb — so the derived subject names the module twice. It is consistent, and it is what the -// catalogue actually publishes, so it is what the controller must listen to. It changes when those -// names are converted, and not before: a subscription written against the name design 29 specifies -// would be a controller listening to a subject nothing publishes. +// **Derived the same way a module's subscription is**, from the emitter and the bare local event +// name, rather than written out. They were written out while the catalogue still spelled its events +// as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a +// hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from +// the other side. Deriving them means the conversion could not leave these behind. var ControllerFollows = []string{ - "mesh.mod.mesh-catalog.event.module.mesh-catalog.upgraded", - "mesh.mod.mesh-catalog.event.module.mesh-catalog.catching-up", + moduleEventSubject("mesh-catalog", "upgraded"), + moduleEventSubject("mesh-catalog", "catching-up"), +} + +// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so +// what the controller subscribes and what the emitter is permitted to publish cannot drift apart. +func moduleEventSubject(module, event string) string { + return "mesh.mod." + module + ".event." + event } // MeshConsumers is what the controller consumes, in the order a person reads it. diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go index dc70bb8..ca450a3 100644 --- a/internal/broker/streams_test.go +++ b/internal/broker/streams_test.go @@ -125,7 +125,6 @@ func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) { want := map[string]Retention{ "CONTROL": RetentionWorkQueue, "NODES": RetentionLastPerSubject, - "BUILDS": RetentionWorkQueue, "EVENTS": RetentionLimits, } got := map[string]Retention{} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 6bcd013..0a34b53 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -23,8 +23,8 @@ accounts { MESH { users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.build.>", "mesh.control.>", "mesh.node.>"] } - subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.build.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.module.mesh-catalog.catching-up", "mesh.mod.mesh-catalog.event.module.mesh-catalog.upgraded"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] } + subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.>"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index bfb3f7e..345a017 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -27,6 +27,17 @@ type Seat struct { // provision may only be held by a module providing it at the seat's scope, and its holder is // what a requirement for that provision resolves to when several modules provide it. Delivers string + // Accepts, Emits and Serves are the protocol of the role, as local verbs — the same three a + // module declares for a seat of its own (novox/hq ADR 0118), and empty for most of these: a seat + // is usually about who does a job and not about what may be said to them. + // + // **Named here so the mesh has no role it cannot describe** (ADR 0121). Without them a build + // machine had three audiences for one outcome and nothing derived a grant for any of them, and an + // event about a role had nowhere to live but the namespace of whichever module held that role + // today — which the bus refuses, because a namespace belongs to who it is named for. + Accepts []string + Emits []string + Serves []string // Decision is the record that made it a seat. Decision string } @@ -53,7 +64,12 @@ var seats = []Seat{ {Name: "mesh-catalog", Scope: ScopeMesh, Decision: "novox/hq ADR 0110"}, {Name: "mesh-npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"}, {Name: "mesh-git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"}, - {Name: "mesh-build-machine", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, + // A build is work submitted to this role, and its outcome is the role's own event (ADR 0121). + // One publish reaches whoever asked, the controller that records it, and the catalogue that + // places it in the module graph — which is what the old bus's shared exchange did for free, and + // what a dedicated build branch was doing a second way. + {Name: "mesh-build-machine", Scope: ScopeNode, + Accepts: []string{"build"}, Emits: []string{"built"}, Decision: "novox/hq ADR 0110"}, {Name: "mesh-dns-port", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, {Name: "mesh-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, {Name: "mesh-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0110"}, @@ -197,3 +213,18 @@ var renamedSeats = map[string]string{ "the-resolver-configuration": "mesh-resolver-configuration", "the-showcase": "mesh-showcase", } + +// SeatsWithAProtocol are the mesh's own seats that say something about what may be said to them or by +// them, which is the set the bus derives streams, consumers and permissions from. +// +// Most of the set is not here, and that is the ordinary case: a seat saying only who does a job grants +// nothing on the bus and needs no queue. +func SeatsWithAProtocol() []Seat { + var out []Seat + for _, s := range seats { + if len(s.Accepts) > 0 || len(s.Emits) > 0 || len(s.Serves) > 0 { + out = append(out, s) + } + } + return out +} diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index a9a6ae1..955394f 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -38,6 +38,15 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) { seats[s.Name] = s } } + // And the mesh's own, which carry protocol too (novox/hq ADR 0121). Added after the modules' + // rather than before, because a `mesh-*` name is the mesh's and registration refuses a module + // declaring one — so this cannot be shadowed, and if it ever were, the mesh's own would win. + for _, own := range catalogue.SeatsWithAProtocol() { + seats[own.Name] = catalogue.SeatDeclaration{ + Name: own.Name, Scope: own.Scope, + Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves, + } + } out := broker.Records{Assigned: map[string][]broker.Declared{}, People: map[string][]string{}} for _, n := range nodes { @@ -88,9 +97,9 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio Serves: m.Tools, } for _, c := range m.Claims { - // A seat the mesh defines for itself carries no protocol, so holding one grants nothing here: - // those seats say who does a job, not who may say what. - if s, declaredSomewhere := seats[c.Name]; declaredSomewhere { + // Every seat with a protocol, the mesh's own included. One that says only who does a job is + // not here and grants nothing, which is most of them. + if s, hasAProtocol := seats[c.Name]; hasAProtocol { d.Holds = append(d.Holds, asSeat(s)) } } @@ -106,6 +115,18 @@ func asSeat(s catalogue.SeatDeclaration) broker.Seat { return broker.Seat{Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves} } +// MeshSeats are the mesh's own seats that carry a protocol, as the bus needs them: what to make a work +// queue for, and whose holder gets a worker on it (novox/hq ADR 0121). +func MeshSeats() []broker.DeclaredSeat { + var out []broker.DeclaredSeat + for _, s := range catalogue.SeatsWithAProtocol() { + out = append(out, broker.DeclaredSeat{ + Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves, + }) + } + return out +} + // NodesWithALiveToken is every machine holding a token that could still be presented — issued, not // expired, not redeemed. // -- 2.54.0 From cc69737934824397003241442b6f84869e26df08 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 15:37:21 +0200 Subject: [PATCH 32/39] The agreement check knows the mesh's own roles, and was passing vacuously without them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It read the roles modules declare and not the mesh's own, so the first consumer of a role's event was skipped as "the emitter is not installed" — which is exactly the silence the check exists to break. It passed, and it was checking nothing. Now it is handed the mesh's own roles too, and there is a case pinning that a consumer of a role event the role does not emit is caught. A check that cannot fail is worse than no check, because it reads as evidence. --- internal/broker/agreement_catalogue_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/broker/agreement_catalogue_test.go b/internal/broker/agreement_catalogue_test.go index 9c4a348..7b9e0ec 100644 --- a/internal/broker/agreement_catalogue_test.go +++ b/internal/broker/agreement_catalogue_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/novox/mesh-controller/internal/catalogue" ) // **Do the catalogue's emitters and consumers agree?** @@ -53,6 +55,14 @@ func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) { t.Fatalf("a wildcard over emitters was reported: %v", bad) } + // A consumer of a role's event whose role does not emit it is caught, which is what stops the + // catalogue check above from passing by knowing nothing about roles. + if bad := Disagreements(nil, + []AConsumer{{Module: "mesh-catalog", Consumes: []string{"mesh-build-machine.finished"}}}, + []DeclaredSeat{{Name: "mesh-build-machine", Emits: []string{"built"}}}); len(bad) != 1 { + t.Fatalf("a consumer of a role event the role does not emit was not caught: %v", bad) + } + // An event published under a seat's name is real even though no module declares it as its own. if bad := Disagreements(nil, []AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}}, @@ -70,7 +80,13 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat } var emitters []AnEmitter var consumers []AConsumer + // The mesh's own roles, which emit under the seat's name rather than any module's (novox/hq + // ADR 0121). Without these the check skips every consumer of a role's event as "the emitter is + // not installed" — which is how it passed vacuously the first time one existed. var seats []DeclaredSeat + for _, own := range catalogue.SeatsWithAProtocol() { + seats = append(seats, DeclaredSeat{Name: own.Name, Accepts: own.Accepts, Emits: own.Emits}) + } for _, e := range entries { if !e.IsDir() { continue -- 2.54.0 From e4e960ec1cfd3ca14ea8874b46bd21a3aaef12ab Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 16:01:53 +0200 Subject: [PATCH 33/39] A build is work submitted to a role, on both buses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0121 carried through to working code. `Builders` is the asking side and `BuildMachine` the taking side, each with an implementation per bus, and the builder binary and the `build` command now go through them. On the bus being built, one publish does what two did. The old bus answered the asker through a reply queue and announced to an events exchange, because two audiences meant two topologies. Here the outcome is the role's own event: the asker matches it by the id its request carried, the controller records it, the catalogue places it in the graph. So a build machine publishes once, needs a reply queue for nothing, and needs a grant over nobody's inbox — which is what ruled out the alternatives. The outcome carries the module name now. Only the manifest says what was built, and on the old bus the separate announcement carried it; with one message for three readers it belongs in the result. A failed build names none, because it produced no module version and the catalogue would otherwise place something that was never made. Checked against a real server: the whole round trip; a third party on the role's event hearing the same outcome the asker did, which is the claim the decision rests on; work leaving the queue once settled, so no second machine repeats it; work submitted with no machine holding the role waiting instead of failing, and being done when one arrives; and work a machine handed back coming round again. One thing I got wrong twice now and have written down where it bit: binding to a consumer must name that consumer's own filter subject, not the narrower subject the caller cares about. The client compares the two and refuses anything that is not equal, with "subject does not match consumer". --- cmd/mesh-builder/main.go | 171 +++++++------------- cmd/mesh-controller/build.go | 35 +++- internal/broker/streams.go | 9 ++ internal/broker/testdata/composed.conf | 2 +- internal/link/build.go | 10 ++ internal/link/builds.go | 100 ++++++++++++ internal/link/builds_current.go | 184 +++++++++++++++++++++ internal/link/builds_nats.go | 206 +++++++++++++++++++++++ internal/link/builds_nats_test.go | 215 +++++++++++++++++++++++++ internal/link/receive_nats.go | 5 + 10 files changed, 820 insertions(+), 117 deletions(-) create mode 100644 internal/link/builds.go create mode 100644 internal/link/builds_current.go create mode 100644 internal/link/builds_nats.go create mode 100644 internal/link/builds_nats_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index f321436..e27d4b3 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -29,10 +29,10 @@ import ( "os/signal" "strings" "syscall" - "time" amqp "github.com/rabbitmq/amqp091-go" + "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/link" ) @@ -115,72 +115,63 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - conn, err := dial(credential) - if err != nil { - // Not quoted back: the URL carries this builder's broker password. - return fmt.Errorf("cannot reach the broker: %w", err) - } - defer conn.Close() - channel, err := conn.Channel() - if err != nil { - return err - } - defer channel.Close() - - if _, err := channel.QueueDeclare(link.BuildQueue, true, false, false, false, nil); err != nil { - return err - } - // One at a time. A build machine that took five requests at once would run five container - // builds against one runtime and finish all of them slower than it would have finished the - // first — and the queue is what shares work between machines, so nothing is lost by it. - if err := channel.Qos(1, 0, false); err != nil { - return err - } - - // Not auto-acknowledged. A request acknowledged on arrival is a build that vanishes if this - // process dies mid-way, with nobody waiting on it ever hearing why. - requests, err := channel.ConsumeWithContext(ctx, link.BuildQueue, "mesh-builder", - false, false, false, false, nil) + machine, err := takeWorkFrom(credential, on) if err != nil { return err } + defer machine.Close() fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry) publisher := builder.Registry{Address: registry, Run: builder.Command} - for { - select { - case <-ctx.Done(): - fmt.Println("stopping") - return nil - case delivery, ok := <-requests: - if !ok { - return fmt.Errorf("the broker closed the connection") - } - answer(ctx, channel, publisher, on, workspace, delivery) - } + return machine.Take(ctx, func(ctx context.Context, work link.Build) { + answer(ctx, publisher, on, workspace, work) + }) +} + +// takeWorkFrom opens this machine's link to whichever bus the mesh is on. +// +// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build +// machine told about both would take work from one and answer on the other, and every log line would +// say it was fine. +func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) { + address, onNATS, err := broker.OnNATS() + if err != nil { + return nil, err } + if err := broker.MustBeOneBus(credential.URL, address); err != nil { + return nil, err + } + if onNATS { + js, err := broker.Dial(address) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus at %s: %w", address, err) + } + return link.MachineOverNATS(js, on), nil + } + + conn, err := dial(credential) + if err != nil { + // Not quoted back: the URL carries this builder's broker password. + return nil, fmt.Errorf("cannot reach the broker: %w", err) + } + channel, err := conn.Channel() + if err != nil { + conn.Close() + return nil, err + } + return link.MachineOverCurrent(conn, channel, on), nil } // answer does one build and says what happened, whichever way it went. -func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher, - on, workspace string, delivery amqp.Delivery) { +func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) { + request := work.Request() - // **First thing, and to stdout.** A build request that arrives and produces no visible line - // until it either finishes or fails is indistinguishable from one that never arrived — which - // cost a long diagnosis against a running mesh, chasing "the handler never fired" when the - // truth was only that the handler said nothing until the end. - fmt.Fprintf(os.Stderr, "a build request arrived (%d bytes)\n", len(delivery.Body)) - - var request link.BuildRequest - if err := json.Unmarshal(delivery.Body, &request); err != nil { - // Unreadable. Acknowledged and dropped rather than requeued: a message this builder - // cannot parse will not become parseable by being delivered again, and requeueing it - // would put it in front of every real request for ever. - fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err) - _ = delivery.Ack(false) - return - } + // **First thing, and to stdout.** A build request that arrives and produces no visible line until + // it either finishes or fails is indistinguishable from one that never arrived — which cost a long + // diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that + // the handler said nothing until the end. + fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository) result := link.BuildResult{ ID: request.ID, Repository: request.Repository, Path: request.Path, @@ -199,8 +190,8 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis var built builder.Result if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a - // build that could not have resolved its dependencies is refused in front of the reason, - // not after a clone that then fails at npm ci. + // build that could not have resolved its dependencies is refused in front of the reason, not + // after a clone that then fails at npm ci. built, err = builder.Build(ctx, builder.Command, publisher, request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, forgeFrom(), @@ -230,67 +221,19 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis } } - body, err := json.Marshal(result) - if err != nil { - fmt.Fprintf(os.Stderr, "cannot report a build: %v\n", err) - _ = delivery.Ack(false) + if err := work.Announce(ctx, result); err != nil { + // Said, not fatal: the build happened. A build reported as failed because announcing it + // failed is a lie about work that was done — and the request stays unsettled below only if + // nothing was said at all, so another machine can try. + fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err) return } - // Always through the exchange, whether or not somebody is waiting. - // - // **Never the default exchange.** Permission there is granted per exchange rather than per - // queue, so a builder allowed to use it could publish into any node's queue — the privilege a - // build machine most obviously should not have. An asker binds its own reply queue to this - // key and filters by correlation; a control plane that records builds is bound to it too, so - // a result nobody asked for is still kept rather than reported into the void. - publishCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - if err := channel.PublishWithContext(publishCtx, link.Exchange, link.KeyBuilt, false, false, - amqp.Publishing{ - ContentType: "application/json", - CorrelationId: result.ID, - Body: body, - }); err != nil { - fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err) + // Settled only once the outcome is away, so a machine that dies before answering leaves the work + // for another rather than losing it. + if err := work.Done(); err != nil { + fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err) } - // **And announced, which is a different act from answering.** The reply goes to whoever asked - // and is correlated to their request; this says to the whole mesh that a module now exists at - // a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build - // nobody asked for still has to be announced, or the graph knows less than the registry does. - // - // Only on success: a failed build produced no module-version, and announcing one would put - // something in the graph that was never made. - if result.Failed == "" && result.Commit != "" { - announced := map[string]any{ - "module": moduleOf(result.Manifest), "commit": result.Commit, - "repository": result.Repository, "path": result.Path, "ref": result.Ref, - "manifest": json.RawMessage(result.Manifest), "against": result.Against, - "made": result.Made, - } - if err := link.EmitEvent(publishCtx, link.OverCurrent{Channel: channel}, link.KeyModuleBuilt, "builder", on, announced); err != nil { - // Said, not fatal: the build happened and was answered. A module the catalogue has not - // heard of is a gap somebody can close; a build reported as failed because announcing - // it failed is a lie about work that was done. - fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err) - } - } - - // Acknowledged only once the answer is away, so a builder that dies before answering leaves - // the request for another machine rather than losing it. - _ = delivery.Ack(false) -} - -// moduleOf reads the module's name out of the manifest it just built, which is the only place it is -// authoritative — the request named a repository and a path, not a module. -func moduleOf(manifest json.RawMessage) string { - var named struct { - Module string `json:"module"` - } - if err := json.Unmarshal(manifest, &named); err != nil { - return "" - } - return named.Module } // packagesFrom is where a build resolves the mesh's own published packages — the SDK above all diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 582b04c..600d36a 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -395,7 +395,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti } fmt.Println() - result, err := link.RequestBuild(ctx, server.Channel(), request, wait) + ask, err := askOver(server) + if err != nil { + return err + } + defer ask.Close() + + result, err := ask.Submit(ctx, request, wait) if err != nil { return err } @@ -472,7 +478,13 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai } defer server.Close() - result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{ + ask, err := askOver(server) + if err != nil { + return err + } + defer ask.Close() + + result, err := ask.Submit(ctx, link.BuildRequest{ ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), Repository: repository, Path: path, Ref: ref, Held: heldBy(ctx), @@ -557,3 +569,22 @@ func heldBy(ctx context.Context) map[string]string { } return routed } + +// askOver opens the way a build is asked for, on whichever bus the mesh is on. +// +// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus +// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus +// being built it dials, because a build request is a one-shot and holds nothing else. +func askOver(server *link.Server) (link.Builders, error) { + address, onNATS, err := broker.OnNATS() + if err != nil { + return nil, err + } + if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), address); err != nil { + return nil, err + } + if onNATS { + return link.BuildsOverNATS(address) + } + return link.BuildsOverCurrent(server.Channel()), nil +} diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 87748fb..9b31266 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -171,6 +171,10 @@ const ControllerName = "controller" var ControllerFollows = []string{ moduleEventSubject("mesh-catalog", "upgraded"), moduleEventSubject("mesh-catalog", "catching-up"), + // A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a + // message on the control branch. Same three audiences, one publish: whoever asked, this, and the + // catalogue. + seatEventSubject("mesh-build-machine", "built"), } // moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so @@ -179,6 +183,11 @@ func moduleEventSubject(module, event string) string { return "mesh.mod." + module + ".event." + event } +// seatEventSubject is where a role's own event lands, derived the same way a holder's permission is. +func seatEventSubject(seat, verb string) string { + return "mesh.seat." + seat + ".event." + verb +} + // MeshConsumers is what the controller consumes, in the order a person reads it. // // **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's, diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 0a34b53..8e44a6e 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] } - subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.>"] } + subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.>", "mesh.seat.mesh-build-machine.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { diff --git a/internal/link/build.go b/internal/link/build.go index 2ebe602..f097233 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -82,6 +82,16 @@ type BuildResult struct { // about the source. On string `json:"on"` + // Module is what was built, read out of the manifest — the only place it is authoritative, since + // a request names a repository and a path. + // + // **Here because one message now reaches three audiences** (novox/hq ADR 0121). On the bus the + // mesh runs on today the answer and the announcement were two publishes to two topologies, so a + // result needed no module name and the announcement carried one. On the bus being built the + // outcome is the role's own event, and the catalogue reading it needs to know what was built. + // Empty on a failed build, which produced no module version. + Module string `json:"module,omitempty"` + // Commit is what was actually built. The mesh records it, which is what makes "is this // current?" answerable without building again. Commit string `json:"commit,omitempty"` diff --git a/internal/link/builds.go b/internal/link/builds.go new file mode 100644 index 0000000..0542f61 --- /dev/null +++ b/internal/link/builds.go @@ -0,0 +1,100 @@ +package link + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// Asking a role to build something, and being told what came of it. +// +// A build is work submitted to a role, not a message to a machine (novox/hq ADR 0121). The +// build-machine seat accepts a build and emits an outcome, so the same publish that answers whoever +// asked also reaches the controller that records it and the catalogue that places it in the module +// graph — and no build machine needs permission to publish into anybody's inbox. +// +// **This is the one flow whose shape differs from every other**, which is why it has its own seam +// rather than living in `Bus`. Everything else the controller sends is either an event nobody must +// act on or a declaration a node reconciles toward; a build is a request that takes minutes and has +// exactly one answer. Too long for request/reply, too particular to be an event. + +// TheBuildMachine is the role a build is submitted to. +const TheBuildMachine = "mesh-build-machine" + +// BuildWork is where a build request lands, and BuildOutcome is where its result does. Derived from +// the seat, so both sides name the role and neither names the other. +func BuildWork() string { return "mesh.seat." + TheBuildMachine + ".accept.build" } +func BuildOutcome() string { return "mesh.seat." + TheBuildMachine + ".event.built" } + +// Builders is how work reaches a build machine and how the outcome comes back. +type Builders interface { + // Submit asks for one build and waits for its outcome. + // + // The wait is long by nature. A build clones, pulls a base image and runs a container build, so + // a timeout here says "nothing is doing builds" rather than "this build is slow" — and the two + // need different remedies, which is why the message distinguishes them. + Submit(ctx context.Context, request BuildRequest, wait time.Duration) (BuildResult, error) + + // Close lets go of whatever was dialled. + Close() +} + +// BuildMachine is a machine taking work from the role it holds. +type BuildMachine interface { + // Take hands each request to do until the context ends, and says why it stopped. + Take(ctx context.Context, do func(context.Context, Build)) error + Close() +} + +// Build is one request a machine has been handed. +type Build interface { + // Request is what to build. + Request() BuildRequest + + // Announce publishes the outcome as the role's own event. + // + // One publish, three audiences: whoever asked matches it by the id their request carried, the + // controller records it, and the catalogue places it. On the bus the mesh runs on today that + // fan-out came from a shared exchange; here the mesh derived the subject. + Announce(ctx context.Context, result BuildResult) error + + // Done settles the request. Called only after the outcome is away, so a machine that dies + // before announcing leaves the work for another rather than losing it. + Done() error + + // Hold hands the work back for another attempt after the delay. + Hold(after time.Duration) error +} + +// waitingFor is the message a caller gets when nothing answered. Its own function because both +// transports say it, and saying it differently in two places is how one of them ends up vague. +func waitingFor(wait time.Duration) error { + return fmt.Errorf( + "no build machine answered within %s. Either nothing holds %s — in which case the work is "+ + "queued and will be done when something does — or a build is taking longer than this", + wait, TheBuildMachine) +} + +// theOutcomeOf reads a result and says whether it is the answer to this request. +func theOutcomeOf(body []byte, id string) (BuildResult, bool, error) { + var result BuildResult + if err := json.Unmarshal(body, &result); err != nil { + return BuildResult{}, false, fmt.Errorf("a build machine answered with something unreadable: %w", err) + } + // Somebody else's build. Skipped rather than returned, because returning it would attribute one + // build's outcome to another's. + return result, result.ID == id, nil +} + +// ModuleOf reads the module's name out of a manifest a build produced, which is the only place it is +// authoritative — a request named a repository and a path, not a module. +func ModuleOf(manifest json.RawMessage) string { + var named struct { + Module string `json:"module"` + } + if err := json.Unmarshal(manifest, &named); err != nil { + return "" + } + return named.Module +} diff --git a/internal/link/builds_current.go b/internal/link/builds_current.go new file mode 100644 index 0000000..1f60d41 --- /dev/null +++ b/internal/link/builds_current.go @@ -0,0 +1,184 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The build flow on the bus the mesh runs on today. +// +// Moved behind the seam rather than changed. The queue, the reply binding and the correlation are +// what they were, because the mesh is running on this. + +// currentBuilds asks for builds over a channel. +type currentBuilds struct{ channel *amqp.Channel } + +// BuildsOverCurrent is the asking side on the bus the mesh has. +func BuildsOverCurrent(channel *amqp.Channel) Builders { return currentBuilds{channel: channel} } + +func (b currentBuilds) Close() {} + +func (b currentBuilds) Submit(ctx context.Context, request BuildRequest, + wait time.Duration) (BuildResult, error) { + + // Its own queue for the answer, declared before the ask. Consuming from the shared exchange + // would mean competing with the controller's own consumer for a message meant for this caller. + replies, err := b.channel.QueueDeclare(ReplyQueue(request.ID), false, true, true, false, nil) + if err != nil { + return BuildResult{}, err + } + // **A builder never publishes to the default exchange**, because permission there is per + // exchange and not per queue — a builder allowed to use it could publish into any node's queue, + // which is the privilege a build machine most obviously should not have. The cost is that every + // asker sees every result, which is why the correlation is checked below rather than assumed. + if err := b.channel.QueueBind(replies.Name, KeyBuilt, Exchange, false, nil); err != nil { + return BuildResult{}, err + } + answers, err := b.channel.ConsumeWithContext(ctx, replies.Name, "", true, true, false, false, nil) + if err != nil { + return BuildResult{}, err + } + + body, err := json.Marshal(request) + if err != nil { + return BuildResult{}, err + } + if err := b.channel.PublishWithContext(ctx, "", BuildQueue, false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + CorrelationId: request.ID, + ReplyTo: replies.Name, + Body: body, + }); err != nil { + return BuildResult{}, err + } + + waiting, cancel := context.WithTimeout(ctx, wait) + defer cancel() + for { + select { + case <-waiting.Done(): + return BuildResult{}, waitingFor(wait) + case delivery, ok := <-answers: + if !ok { + return BuildResult{}, errors.New("the connection closed while waiting for a build") + } + result, mine, err := theOutcomeOf(delivery.Body, request.ID) + if err != nil { + return BuildResult{}, err + } + if mine { + return result, nil + } + } + } +} + +// --- the machine's side --------------------------------------------------------------------- + +type currentMachine struct { + conn *amqp.Connection + channel *amqp.Channel + on string +} + +// MachineOverCurrent takes build work over a channel. +func MachineOverCurrent(conn *amqp.Connection, channel *amqp.Channel, on string) BuildMachine { + return ¤tMachine{conn: conn, channel: channel, on: on} +} + +func (m *currentMachine) Close() {} + +func (m *currentMachine) Take(ctx context.Context, do func(context.Context, Build)) error { + if _, err := m.channel.QueueDeclare(BuildQueue, true, false, false, false, nil); err != nil { + return err + } + // One at a time. A machine that took five requests at once would run five container builds + // against one runtime and finish all of them slower than it would have finished the first — and + // the queue is what shares work between machines, so nothing is lost by it. + if err := m.channel.Qos(1, 0, false); err != nil { + return err + } + // Not auto-acknowledged: a request acknowledged on arrival is a build that vanishes if this + // process dies mid-way, with nobody waiting on it ever hearing why. + requests, err := m.channel.ConsumeWithContext(ctx, BuildQueue, "mesh-builder", + false, false, false, false, nil) + if err != nil { + return err + } + for { + select { + case <-ctx.Done(): + return nil + case delivery, ok := <-requests: + if !ok { + return errors.New("the broker closed the connection") + } + var request BuildRequest + if err := json.Unmarshal(delivery.Body, &request); err != nil { + // Unreadable: rejected rather than retried, because the next attempt reads the same + // bytes. Nobody waiting hears an answer, which is correct — there was no request. + _ = delivery.Reject(false) + continue + } + do(ctx, ¤tBuild{request: request, delivery: delivery, on: m.on, channel: m.channel}) + } + } +} + +type currentBuild struct { + request BuildRequest + delivery amqp.Delivery + on string + channel *amqp.Channel +} + +func (b *currentBuild) Request() BuildRequest { return b.request } + +// Announce answers and announces, which on this bus are two publishes to two exchanges. +// +// The reply goes to whoever asked, correlated to their request; the announcement says to the whole +// mesh that a module now exists at a commit (novox/hq ADR 0072). Only a successful build is +// announced: a failed one produced no module version, and announcing one would put something in the +// graph that was never made. +func (b *currentBuild) Announce(ctx context.Context, result BuildResult) error { + body, err := json.Marshal(result) + if err != nil { + return err + } + if err := b.channel.PublishWithContext(ctx, Exchange, KeyBuilt, false, false, amqp.Publishing{ + ContentType: "application/json", + CorrelationId: result.ID, + Body: body, + }); err != nil { + return fmt.Errorf("cannot answer a build request: %w", err) + } + if result.Failed != "" || result.Commit == "" { + return nil + } + return EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyModuleBuilt, "builder", b.on, + announcementOf(result)) +} + +func (b *currentBuild) Done() error { return b.delivery.Ack(false) } + +func (b *currentBuild) Hold(time.Duration) error { + // No delayed redelivery on this bus: handed back at once, which is what it has always done. + return b.delivery.Nack(false, true) +} + +// announcementOf is what the mesh is told about a finished build. One function, so the two +// transports cannot describe the same build differently. +func announcementOf(result BuildResult) map[string]any { + return map[string]any{ + "module": ModuleOf(result.Manifest), "commit": result.Commit, + "repository": result.Repository, "path": result.Path, "ref": result.Ref, + "manifest": json.RawMessage(result.Manifest), "against": result.Against, + "made": result.Made, + } +} diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go new file mode 100644 index 0000000..039d340 --- /dev/null +++ b/internal/link/builds_nats.go @@ -0,0 +1,206 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The build flow on the bus being built. +// +// **One publish where the old bus needed two** (novox/hq ADR 0121). There, the answer went to a +// reply queue and the announcement to an events exchange, because the two audiences were reached by +// two topologies. Here the outcome is the role's own event: whoever asked matches it by the id their +// request carried, the controller records it, the catalogue places it in the graph. So a build +// machine publishes once and needs permission for nothing but its own role's subjects — no reply +// queue to declare, and no grant over anybody's inbox. + +// natsBuilds asks for builds over a connection. +type natsBuilds struct { + js *broker.JetStream + owned bool +} + +// BuildsOverNATS is the asking side on the bus being built. It dials, because the command that asks +// for a build is a one-shot and holds nothing else. +func BuildsOverNATS(address string) (Builders, error) { + js, err := broker.Dial(address) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus at %s to ask for a build: %w", address, err) + } + return &natsBuilds{js: js, owned: true}, nil +} + +func (b *natsBuilds) Close() { + if b.owned && b.js != nil { + b.js.Close() + } +} + +func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, + wait time.Duration) (BuildResult, error) { + + // Subscribed before the ask, so an outcome cannot arrive before there is anywhere for it to + // land. Core, not the stream: the asker is waiting now, and the durable copy of this outcome is + // the same event on EVENTS, which the controller records. + outcomes, err := b.js.Conn().SubscribeSync(BuildOutcome()) + if err != nil { + return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err) + } + defer func() { _ = outcomes.Unsubscribe() }() + if err := b.js.Conn().Flush(); err != nil { + return BuildResult{}, err + } + + body, err := json.Marshal(request) + if err != nil { + return BuildResult{}, err + } + // Into the role's work queue and awaited: work the bus never accepted must fail here rather than + // be assumed, because nothing else will ever say so. + publish, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil { + return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err) + } + + waiting, cancelWait := context.WithTimeout(ctx, wait) + defer cancelWait() + for { + msg, err := outcomes.NextMsgWithContext(waiting) + switch { + case errors.Is(err, context.DeadlineExceeded): + return BuildResult{}, waitingFor(wait) + case errors.Is(err, context.Canceled): + return BuildResult{}, ctx.Err() + case err != nil: + return BuildResult{}, fmt.Errorf("waiting for a build's outcome: %w", err) + } + result, mine, err := theOutcomeOf(msg.Data, request.ID) + if err != nil { + return BuildResult{}, err + } + if mine { + return result, nil + } + } +} + +// --- the machine's side --------------------------------------------------------------------- + +type natsMachine struct { + js *broker.JetStream + on string + seat string + sub *nats.Subscription +} + +// MachineOverNATS takes build work from the role this machine holds. +func MachineOverNATS(js *broker.JetStream, on string) BuildMachine { + return &natsMachine{js: js, on: on, seat: TheBuildMachine} +} + +func (m *natsMachine) Close() { + if m.sub != nil { + _ = m.sub.Unsubscribe() + } +} + +// Take binds to the role's worker and hands each request over, one at a time. +// +// **Bound, never created.** The work queue and the worker on it are the controller's to define +// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said +// as the mesh's to answer rather than quietly created with whatever this client defaults to. +func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error { + worker, found := broker.HolderConsumerFor(m.on, "builder", + broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}}) + if !found { + return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat) + } + + // One at a time, which the consumer's own ack-pending limit enforces rather than a prefetch + // setting: a machine that took five requests at once would run five container builds against one + // runtime and finish all of them slower than the first. + work := make(chan *nats.Msg, 1) + // **The consumer's own filter, not the one subject this machine cares about.** The client checks + // what is asked for against the consumer's filter and refuses anything that is not the same — + // "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered + // on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different + // consumers, which is why it is written down here. + filter := worker.Filters[0] + sub, err := m.js.Context().ChanQueueSubscribe(filter, worker.Queue, work, + nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + return fmt.Errorf( + "this machine cannot take work from %s: %w. The mesh creates that queue and this "+ + "machine's worker on it, and a build machine may not create one itself — so this is "+ + "the mesh's to answer, not this machine's", m.seat, err) + } + m.sub = sub + + for { + select { + case <-ctx.Done(): + return nil + case msg, ok := <-work: + if !ok { + return errors.New("the bus stopped delivering build work") + } + var request BuildRequest + if err := json.Unmarshal(msg.Data, &request); err != nil { + // Unreadable: terminated rather than retried, because the next attempt reads the same + // bytes. Nobody waiting hears an answer, which is right — there was no request. + _ = msg.Term() + continue + } + do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js}) + } + } +} + +type natsBuild struct { + request BuildRequest + msg *nats.Msg + on string + js *broker.JetStream +} + +func (b *natsBuild) Request() BuildRequest { return b.request } + +// Announce publishes the outcome as the role's own event, once, for all three audiences. +// +// Into the stream, so a controller that was restarting still records it and a catalogue that was +// down still catches up. The asker is listening on core for the same subject and gets it either way: +// a stream delivers to its durable consumers and the plain subscribers both. +func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error { + // One body, three readers. Whoever asked matches the id; the controller records it; the catalogue + // needs to know what was built, which only the manifest says — so the result carries it rather + // than a second message carrying a second shape. + // + // **A failed build names no module**, because it produced no module version and the catalogue + // would otherwise put something in the graph that was never made. The asker still gets its + // answer: a failure is the answer. + if result.Failed == "" && result.Commit != "" { + result.Module = ModuleOf(result.Manifest) + } + body, err := json.Marshal(result) + if err != nil { + return err + } + publish, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if _, err := b.js.Context().Publish(BuildOutcome(), body, nats.Context(publish)); err != nil { + return fmt.Errorf("cannot announce a build's outcome: %w", err) + } + return nil +} + +func (b *natsBuild) Done() error { return b.msg.Ack() } + +func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) } diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go new file mode 100644 index 0000000..2672258 --- /dev/null +++ b/internal/link/builds_nats_test.go @@ -0,0 +1,215 @@ +package link + +import ( + "context" + "encoding/json" + "io" + "log" + "os" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// A build, end to end, against a real server. +// +// The claim worth checking is the one ADR 0121 rests on: **one publish reaches three audiences**. +// Whoever asked matches the outcome by the id their request carried; the controller records it; the +// catalogue places it. On the old bus that fan-out came from a shared exchange, and it would be easy +// to write a version where only the asker hears it and nobody notices for weeks. +// +// docker run -d --rm --name t -p 14230:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14230 go test ./internal/link/ -run TestNatsABuild + +func aBusWithTheBuildRole(t *testing.T) *broker.JetStream { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + + seats := []broker.DeclaredSeat{{Name: TheBuildMachine, Accepts: []string{"build"}, + Emits: []string{"built"}}} + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := broker.RaiseSeats(js, seats, map[string]broker.Holder{ + TheBuildMachine: {Node: "anchor", Module: "builder"}, + }); err != nil { + t.Fatal(err) + } + clean := func() { + _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") + for _, s := range broker.MeshStreams() { + _ = js.Context().PurgeStream(s.Name) + } + } + t.Cleanup(clean) + return js +} + +// The whole round trip: asked, taken, built, and the outcome heard by the asker and by a consumer of +// the role's event who never asked for anything. +func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + // A third party on the role's event — what the catalogue is. Subscribed first, so nothing is + // missed. + heard := make(chan BuildResult, 4) + watching, err := js.Conn().Subscribe(BuildOutcome(), func(msg *nats.Msg) { + var r BuildResult + if json.Unmarshal(msg.Data, &r) == nil { + heard <- r + } + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = watching.Unsubscribe() }() + _ = js.Conn().Flush() + + // A build machine holding the role. + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + failed := make(chan error, 1) + go func() { + failed <- machine.Take(ctx, func(ctx context.Context, work Build) { + r := work.Request() + _ = work.Announce(ctx, BuildResult{ + ID: r.ID, Repository: r.Repository, On: "anchor", Commit: "abc1234", + Manifest: json.RawMessage(`{"module":"shop"}`), + }) + _ = work.Done() + }) + }() + + ask := &natsBuilds{js: js} + result, err := ask.Submit(ctx, BuildRequest{ID: "b-1", Repository: "/r"}, 15*time.Second) + if err != nil { + select { + case why := <-failed: + t.Fatalf("the machine could not take work: %v", why) + default: + } + t.Fatalf("the asker never got an outcome: %v", err) + } + if result.ID != "b-1" || result.Commit != "abc1234" { + t.Fatalf("the asker got %+v", result) + } + // Named in the outcome, because only the manifest says what was built and the catalogue reading + // this event needs to know. + if result.Module != "shop" { + t.Errorf("the outcome names module %q, so a catalogue reading it cannot place the build", + result.Module) + } + + select { + case also := <-heard: + if also.ID != "b-1" { + t.Fatalf("a third party heard %+v", also) + } + case <-time.After(5 * time.Second): + t.Fatal("nobody but the asker heard the outcome, so the catalogue would never place the build") + } + + // And the work left the queue: a request a machine took and settled must not be given to another. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE") + if err == nil && info.State.Msgs == 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the request is still queued after being settled, so another machine would build it again") +} + +// Work submitted with no machine holding the role waits rather than failing, and is done when one +// arrives. **That is what a queue is for**, and the alternative — refusing because nobody is there +// yet — would make installing a build machine an ordering problem. +func TestNatsABuildWaitsForAMachineRatherThanFailing(t *testing.T) { + js := aBusWithTheBuildRole(t) + + body, _ := json.Marshal(BuildRequest{ID: "b-2", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE") + if err != nil || info.State.Msgs != 1 { + t.Fatalf("the work did not queue: %+v %v", info, err) + } + + // Now a machine arrives and finds it waiting. + ctx, stop := context.WithCancel(context.Background()) + defer stop() + took := make(chan string, 1) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-took: + if id != "b-2" { + t.Fatalf("the machine took %q", id) + } + case <-time.After(10 * time.Second): + t.Fatal("a machine that arrived after the work did never got it, so the backlog was lost") + } +} + +// A machine that dies before saying anything leaves the work for another. +func TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + body, _ := json.Marshal(BuildRequest{ID: "b-3", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + + var once sync.Once + handed := make(chan struct{}, 2) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + handed <- struct{}{} + // The first time, hand it straight back — a machine that stopped mid-build. + var settled bool + once.Do(func() { _ = work.Hold(200 * time.Millisecond); settled = true }) + if !settled { + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"}) + _ = work.Done() + } + }) + }() + + for i := 0; i < 2; i++ { + select { + case <-handed: + case <-time.After(10 * time.Second): + t.Fatalf("the work was handed over %d time(s); unanswered work must come back", i) + } + } +} + +func quietLog() *log.Logger { return log.New(io.Discard, "", 0) } + +var _ = quietLog diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go index 3acac49..24ab322 100644 --- a/internal/link/receive_nats.go +++ b/internal/link/receive_nats.go @@ -180,6 +180,11 @@ func kindOfSubject(subject string) (string, bool) { return KindModuleMoved, true case broker.ControllerFollows[1]: return KindCatchUp, true + case BuildOutcome(): + // A build's outcome is the role's event now, so it arrives on the events stream rather than + // the control branch — and is acted on by the same handler, because what the controller does + // with it did not change (novox/hq ADR 0121). + return KindBuilt, true } return "", false } -- 2.54.0 From 8e2824201a0819b1f3560f96064d9cb68599a81e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 16:39:19 +0200 Subject: [PATCH 34/39] Genesis can raise a mesh on the new bus, and the carried user list is checked against the composer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mesh writes its own user list, and at genesis there is no mesh yet to write it. So the installer carries the first one — the controller's own account at a well-known bootstrap password, exactly as the store is reached at `postgres:bootstrap` and the old bus at `guest:guest`, and rotated with them. From the controller's first composition onward the file is the controller's. That left a gap I would not have found by reading: the controller's own account is created before there is a controller to mint one, so nothing recorded a hash for it, and its first composition would have left the writer out of the file it was writing — a bus nothing can connect to, produced by the thing connected to it. It now records a hash of the credential it is actually using, and only if none is recorded, so a restart cannot put the bootstrap password back over a rotated one. The carried list and the derived one are two statements of one fact, so a test compares them: every subject the controller derives must be in the template, and nothing wider. It earned itself immediately — the composer was granting both a role's whole event branch and the one event it actually follows, which is a wider way of saying the same thing, and the wider one wins. Only the submitting half of a role is granted now; what comes back is named exactly. Getting this wrong is the worst kind of silent. A controller whose carried permissions are narrower than the ones it derives comes up, connects, and is refused on the first thing it tries, with an authorisation error naming a subject and not the template that forgot it — and a mesh cannot be raised twice to find out. --- cmd/mesh-controller/push.go | 17 +++ internal/broker/genesis_template_test.go | 126 +++++++++++++++++++++++ internal/broker/nats.go | 1 - internal/broker/onnats.go | 20 ++++ internal/broker/onnats_test.go | 20 ++++ internal/broker/testdata/composed.conf | 2 +- internal/inventory/bususers.go | 27 +++++ internal/inventory/bususers_test.go | 37 +++++++ 8 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 internal/broker/genesis_template_test.go diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index f499281..71cee3b 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -702,6 +702,23 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) } defer js.Close() + // **Its own user, before anything else.** The controller's account is created by the installer at + // a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for + // it, and the first composition would leave the writer out of the file it was writing. Recorded + // only if absent: a credential the mesh minted since is the one that counts. + // **Its own user, before anything else it does here.** The controller's account is created by the + // installer at a bootstrap password, before there is a controller to mint one — so nothing + // recorded a hash for it, and the first composition would leave the writer out of the file it was + // writing: a bus nothing can connect to, produced by the thing connected to it. Recorded only if + // absent, so a restart cannot put the bootstrap credential back over a rotated one. + if user, password, _ := broker.CredentialIn(address); user != "" && password != "" { + if err := inv.SeedBusUser(ctx, inventory.BusUser{ + Username: user, Kind: inventory.BusController, + }, password); err != nil { + return fmt.Errorf("cannot record the credential this control plane is using: %w", err) + } + } + nodes, err := inv.Nodes(ctx) if err != nil { return err diff --git a/internal/broker/genesis_template_test.go b/internal/broker/genesis_template_test.go new file mode 100644 index 0000000..1260f85 --- /dev/null +++ b/internal/broker/genesis_template_test.go @@ -0,0 +1,126 @@ +package broker + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "golang.org/x/crypto/bcrypt" +) + +// **The first user list the installer carries must be the one the controller would compose.** +// +// At genesis there is no mesh to write the bus's user list, so the installer carries one: the +// controller's own account, at a bootstrap password, the way the store is reached at +// `postgres:bootstrap` (novox/hq design 25 §4, task 1.7). It is written by hand in a template and +// derived in code here, which is two statements of one fact — so this compares them. +// +// Getting it wrong is the worst kind of silent: a controller whose carried permissions are narrower +// than the ones it derives comes up, connects, and is refused on the first thing it tries, with an +// authorisation error that names a subject and not the template that forgot it. And a mesh cannot be +// raised twice to find out. +func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T) { + accounts := theCarriedAccounts(t) + + want, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + carriedPub := subjectsIn(accounts, "publish") + carriedSub := subjectsIn(accounts, "subscribe") + + if diff := missing(want.Publish, carriedPub); len(diff) > 0 { + t.Errorf("the installer's user list does not let the controller publish %v — it would come up "+ + "and be refused on the first thing it tried", diff) + } + if diff := missing(want.Subscribe, carriedSub); len(diff) > 0 { + t.Errorf("the installer's user list does not let the controller subscribe %v", diff) + } + // And nothing wider than what it derives, or genesis quietly grants a privilege the composition + // takes away again on the first push. + if diff := missing(carriedPub, want.Publish); len(diff) > 0 { + t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff) + } + if diff := missing(carriedSub, want.Subscribe); len(diff) > 0 { + t.Errorf("the installer's user list lets the controller subscribe %v, which it does not derive", diff) + } + + // The credential is the bootstrap one and the hash really is of it, because a hash of something + // else is a controller that cannot log in to the bus it was just given. + hash := regexp.MustCompile(`\$2[aby]?\$[0-9]+\$[A-Za-z0-9./]{53}`).FindString(accounts) + if hash == "" { + t.Fatal("the installer's user list carries no password hash") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { + t.Fatalf("the carried hash does not verify the bootstrap credential the template also carries: %v", err) + } +} + +// theCarriedAccounts is the accounts file the installer's template writes at genesis. +func theCarriedAccounts(t *testing.T) string { + t.Helper() + path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock") + raw, err := os.ReadFile(path) + if err != nil { + t.Skipf("the host's checkout is not beside this one: %v", err) + } + // The template is JSON with line comments, which is how every one of them is written. + var lines []string + for _, l := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(l), "//") { + lines = append(lines, l) + } + } + var bundle struct { + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil { + t.Fatalf("the template is not readable: %v", err) + } + for _, r := range bundle.Resources { + if r["id"] == "bus-accounts" { + content, _ := r["content"].(string) + if content == "" { + t.Fatal("the template's accounts file is empty, so the bus would refuse every connection") + } + return content + } + } + t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use") + return "" +} + +// subjectsIn reads one allow-list out of a composed accounts file. +func subjectsIn(accounts, which string) []string { + found := regexp.MustCompile(which + `: \{ allow: \[([^\]]*)\]`).FindStringSubmatch(accounts) + if len(found) != 2 { + return nil + } + var out []string + for _, part := range strings.Split(found[1], ",") { + if s := strings.Trim(strings.TrimSpace(part), `"`); s != "" { + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +// missing is what is in want and not in got. +func missing(want, got []string) []string { + have := map[string]bool{} + for _, g := range got { + have[g] = true + } + var out []string + for _, w := range want { + if !have[w] { + out = append(out, w) + } + } + return out +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 45da6b0..295f2e4 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -167,7 +167,6 @@ func PermissionsFor(p Principal) (Permissions, error) { // like the catalogue does — which is why no holder needs to publish into anybody's inbox. for _, seat := range meshSeatsTheControllerUses { pub = append(pub, "mesh.seat."+seat+".accept.>") - sub = append(sub, "mesh.seat."+seat+".event.>") } // The two events it reacts to, and its ack subject on the stream they arrive from diff --git a/internal/broker/onnats.go b/internal/broker/onnats.go index f6efa54..12d81ed 100644 --- a/internal/broker/onnats.go +++ b/internal/broker/onnats.go @@ -35,6 +35,26 @@ func OnNATS() (address string, on bool, err error) { return address, true, nil } +// CredentialIn reads the user and password out of a bus address, and the address without them. +// +// The controller's own credential arrives in its address, the way the old bus's does. Split out so the +// controller can record a hash of what it is actually using: its user is created by the installer at a +// bootstrap password, before the controller exists to mint one, and a composition that left itself out +// would produce a bus the writer cannot connect to. +func CredentialIn(address string) (user, password, bare string) { + at := strings.LastIndex(address, "@") + if at < 0 { + return "", "", address + } + scheme := "" + rest := address[:at] + if i := strings.Index(rest, "://"); i >= 0 { + scheme, rest = rest[:i+3], rest[i+3:] + } + user, password, _ = strings.Cut(rest, ":") + return user, password, scheme + address[at+1:] +} + // MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic. // // **Both clients ship and that is the point; both being live is not.** The rollout moves every node diff --git a/internal/broker/onnats_test.go b/internal/broker/onnats_test.go index 19e78e9..f21bc87 100644 --- a/internal/broker/onnats_test.go +++ b/internal/broker/onnats_test.go @@ -38,3 +38,23 @@ func TestOneBusOrNeitherIsAllowed(t *testing.T) { } } } + +// The controller's own credential arrives in its address, and has to be readable out of it — its user +// is created by the installer at a bootstrap password, before the controller exists to mint one. +func TestACredentialIsReadOutOfABusAddress(t *testing.T) { + for _, c := range []struct{ in, user, password, bare string }{ + {"nats://controller:secret@127.0.0.1:4222", "controller", "secret", "nats://127.0.0.1:4222"}, + {"controller:secret@127.0.0.1:4222", "controller", "secret", "127.0.0.1:4222"}, + {"nats://127.0.0.1:4222", "", "", "nats://127.0.0.1:4222"}, + {"127.0.0.1:4222", "", "", "127.0.0.1:4222"}, + // A password containing an at-sign: split on the last one, or the address becomes part of the + // credential and the connection goes somewhere nobody named. + {"nats://controller:a@b@127.0.0.1:4222", "controller", "a@b", "nats://127.0.0.1:4222"}, + } { + user, password, bare := CredentialIn(c.in) + if user != c.user || password != c.password || bare != c.bare { + t.Errorf("%q read as %q/%q at %q; wanted %q/%q at %q", + c.in, user, password, bare, c.user, c.password, c.bare) + } + } +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 8e44a6e..f64b2e8 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -24,7 +24,7 @@ accounts { users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] } - subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.>", "mesh.seat.mesh-build-machine.event.built"] } + subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { diff --git a/internal/inventory/bususers.go b/internal/inventory/bususers.go index c9a43a7..780a193 100644 --- a/internal/inventory/bususers.go +++ b/internal/inventory/bususers.go @@ -136,3 +136,30 @@ func (i *Inventory) ForgetBusUsersOf(ctx context.Context, node string) error { _, err := i.store.Pool().Exec(ctx, `delete from bus_user where node = $1`, node) return err } + +// SeedBusUser records a hash of a credential the mesh did not mint, so a composition contains it. +// +// **Genesis is the reason this exists.** The controller's own user is created before the controller +// runs — by the installer, at a well-known bootstrap password, the way the store's and the old bus's +// are (`postgres:bootstrap`, `guest:guest`). Nothing minted it, so nothing recorded a hash for it, and +// the controller's first composition would leave itself out of the very file it was writing: a bus +// nothing can connect to, produced by the thing connected to it. +// +// Idempotent, and it does not overwrite. A credential the mesh *did* mint is the one that counts, so +// once there is a row this does nothing — otherwise a restart would put the bootstrap password back +// over a rotated one. +func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string) error { + if u.Username == "" || u.Kind == "" || password == "" { + return errors.New("a bus user needs a username, a kind and the credential it is using") + } + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return fmt.Errorf("cannot hash a bus password: %w", err) + } + _, err = i.store.Pool().Exec(ctx, + `insert into bus_user (username, kind, node, module, password_hash) + values ($1, $2, $3, $4, $5) + on conflict (username) do nothing`, + u.Username, u.Kind, u.Node, u.Module, string(hash)) + return err +} diff --git a/internal/inventory/bususers_test.go b/internal/inventory/bususers_test.go index 7af4523..199b660 100644 --- a/internal/inventory/bususers_test.go +++ b/internal/inventory/bususers_test.go @@ -121,3 +121,40 @@ func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { t.Fatal("forgetting the bus users of no node was allowed") } } + +// The controller's own user is created by the installer, so the mesh has to be able to record a +// credential it did not mint — or the first composition leaves the writer out of the file it writes. +func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, known, err := inv.BusUserHash(ctx, "controller") + if err != nil || !known { + t.Fatalf("the credential was not recorded: %v %v", known, err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { + t.Fatalf("what was recorded does not verify the credential given: %v", err) + } + + // Minted since, then seeded again — which is what a restart does. The rotation must stand, or + // every restart would put the bootstrap password back over it. + minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController}) + if err != nil { + t.Fatal(err) + } + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, _, err = inv.BusUserHash(ctx, "controller") + if err != nil { + t.Fatal(err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil { + t.Fatal("a restart put the bootstrap credential back over a rotated one") + } +} -- 2.54.0 From 53e8f5bdd8463b0dcb9d0bec3d845978d706d3cc Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 17:07:19 +0200 Subject: [PATCH 35/39] A person may be issued, listed and revoked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 25 §7's first item, which existed as a permission model and as nothing a person could actually be given. There is a record now, and three commands. Their authority is a list of tools and nothing else. Not a module: they hold no seat, nothing is addressed to them, nothing is delivered to them, and they have no consumer to acknowledge. What they have is permission to ask — which is why there is no scope and no node in the record. Stating what somebody may call replaces what was there rather than adding to it: a list that could only grow is a permission nobody can take back. Forgetting somebody takes their credential with them, because a person's row gone with their bus user left behind is a credential that still works and that nothing derives — the worst of both, since it keeps working and nobody can explain why. The credential is printed once and the mesh keeps only a hash, the same contract a token has. And it starts working at the next composition rather than immediately, because the bus's users are a file — said out loud in both the issue and the revoke messages, since "revoked" that still works for another minute is worth knowing about. Four properties held by test, each a way of being wrong that would not announce itself: a person may publish exactly the tool subjects they were given and nothing on control, nodes or events; they cannot answer a request; changing the list removes what is no longer named; and forgetting them revokes them. --- cmd/mesh-controller/operator.go | 143 +++++++++++++++++- internal/inventory/busrecords.go | 9 +- internal/inventory/bususers.go | 67 ++++++++ .../0034-a-person-may-reach-the-mesh.sql | 19 +++ internal/inventory/people_test.go | 120 +++++++++++++++ 5 files changed, 355 insertions(+), 3 deletions(-) create mode 100644 internal/inventory/migrations/0034-a-person-may-reach-the-mesh.sql create mode 100644 internal/inventory/people_test.go diff --git a/cmd/mesh-controller/operator.go b/cmd/mesh-controller/operator.go index af037eb..0bf944d 100644 --- a/cmd/mesh-controller/operator.go +++ b/cmd/mesh-controller/operator.go @@ -2,12 +2,15 @@ package main import ( "context" + "encoding/json" "errors" "flag" "fmt" "os" "strings" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/secrets" ) @@ -26,9 +29,25 @@ import ( // operator key make [--out ] make a keypair: private half to the file, public half printed // operator key set tell the mesh which key to seal to // operator key show the public key, its fingerprint, and what it can recover -const operatorUsage = "operator key make [--out ] | operator key set [--replace] | operator key show" +const operatorUsage = "operator key make [--out ] | operator key set [--replace] | " + + "operator key show | operator issue --invokes | operator revoke | " + + "operator list" func operatorCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New(operatorUsage) + } + // The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because + // both answer "who, other than a machine, may do something here" — and a person reading this + // command's usage is asking exactly that. + switch args[0] { + case "issue": + return personIssue(ctx, args[1:]) + case "revoke": + return personRevoke(ctx, args[1:]) + case "list": + return personList(ctx) + } if len(args) < 2 || args[0] != "key" { return errors.New(operatorUsage) } @@ -160,3 +179,125 @@ func readPrivateKey(path string) (string, error) { } return strings.TrimSpace(string(raw)), nil } + +// personIssue gives somebody a credential for the mesh's tools, and prints it once. +// +// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the +// credential exists anywhere but on the workstation that will use it — the same contract a token has, +// and for the same reason: a credential recoverable from the mesh's store has the store's blast +// radius. +func personIssue(ctx context.Context, args []string) error { + set := flag.NewFlagSet("operator issue", flag.ContinueOnError) + invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 1 { + return errors.New("operator issue --invokes ") + } + name := set.Arg(0) + if *invokes == "" { + return errors.New( + "say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " + + "or --invokes '*' for an administrator") + } + var tools []string + for _, t := range strings.Split(*invokes, ",") { + if t = strings.TrimSpace(t); t != "" { + tools = append(tools, t) + } + } + + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil { + return err + } + // Refused here rather than at the next composition, where it would stop the whole file being + // written for everybody. A name that cannot be part of a subject is one the server would read as + // a wider permission than anybody granted. + if _, err := broker.PermissionsFor(broker.Principal{ + Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x", + }); err != nil { + return err + } + + user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username() + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson}) + if err != nil { + return err + } + + where, err := broker.FromEnvironment() + if err != nil && !errors.Is(err, broker.ErrNotConfigured) { + return err + } + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + User string `json:"user"` + Password string `json:"password"` + Person string `json:"person"` + Invokes []string `json:"invokes"` + }{ + URL: "nats://" + where.Address, Fingerprint: where.Fingerprint, + User: user, Password: password, Person: name, Invokes: tools, + }) + if err != nil { + return err + } + + fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", ")) + fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash") + fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker") + fmt.Println() + fmt.Println(string(held)) + return nil +} + +func personRevoke(ctx context.Context, args []string) error { + if len(args) != 1 { + return errors.New("operator revoke ") + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + + if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil { + return err + } + // **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops + // working when the file no longer names it. Said plainly, because "revoked" that still works for + // another minute is worth knowing about. + fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+ + "push the machine holding mesh-broker to make it so\n", args[0]) + return nil +} + +func personList(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + + people, err := open.inventory.People(ctx) + if err != nil { + return err + } + if len(people) == 0 { + fmt.Println("nobody but machines reaches this mesh") + return nil + } + for _, p := range people { + fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", ")) + } + return nil +} diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index 955394f..4b29bef 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -80,8 +80,13 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) { } out.Enrolling = enrolling - // People are not recorded yet: the account model is built (design 25 §7's first item) and - // `operator issue` is not, so there is nobody to derive. Left empty rather than guessed at. + people, err := i.People(ctx) + if err != nil { + return broker.Records{}, err + } + for _, p := range people { + out.People[p.Name] = p.Invokes + } return out, nil } diff --git a/internal/inventory/bususers.go b/internal/inventory/bususers.go index 780a193..b46a643 100644 --- a/internal/inventory/bususers.go +++ b/internal/inventory/bususers.go @@ -163,3 +163,70 @@ func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string) u.Username, u.Kind, u.Node, u.Module, string(hash)) return err } + +// A person who may call the mesh's tools (novox/hq design 25 §7). +// +// **Their authority is a list of tools and nothing else.** Not a module: they hold no seat, nothing is +// addressed to them, nothing is delivered to them, and they have no consumer to acknowledge. What +// they have is permission to ask. + +// Person is somebody who may reach the mesh's tools. +type Person struct { + Name string + // Invokes are the tools they may call, each `.`, or the single entry `*` for an + // administrator. + Invokes []string +} + +// RecordPerson adds somebody, or changes what they may call. +// +// Replacing rather than merging: what a person may call is stated in full, so a change that meant to +// remove a tool does remove it. A list that could only grow is a permission nobody can take back. +func (i *Inventory) RecordPerson(ctx context.Context, p Person) error { + if p.Name == "" { + return errors.New("a person needs a name: it becomes their user on the bus") + } + if len(p.Invokes) == 0 { + return fmt.Errorf( + "%s may call nothing, so there is no reason for them to reach the mesh. Name the tools, "+ + "or `*` for an administrator", p.Name) + } + _, err := i.store.Pool().Exec(ctx, + `insert into person (name, invokes) values ($1, $2) + on conflict (name) do update set invokes = excluded.invokes`, + p.Name, p.Invokes) + return err +} + +// People is everybody who may reach the mesh's tools. +func (i *Inventory) People(ctx context.Context) ([]Person, error) { + rows, err := i.store.Pool().Query(ctx, `select name, invokes from person order by name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Person + for rows.Next() { + var p Person + if err := rows.Scan(&p.Name, &p.Invokes); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// ForgetPerson removes somebody and the credential they were given. +// +// **Both, or neither is a revocation.** A person's row gone and their bus user left behind is a +// credential that still works and that nothing derives, which is the worst of both: it keeps working +// and nobody can explain why. +func (i *Inventory) ForgetPerson(ctx context.Context, name string) error { + if name == "" { + return errors.New("forgetting nobody would forget everybody") + } + if _, err := i.store.Pool().Exec(ctx, `delete from person where name = $1`, name); err != nil { + return err + } + return i.ForgetBusUser(ctx, "person."+name) +} diff --git a/internal/inventory/migrations/0034-a-person-may-reach-the-mesh.sql b/internal/inventory/migrations/0034-a-person-may-reach-the-mesh.sql new file mode 100644 index 0000000..83b3082 --- /dev/null +++ b/internal/inventory/migrations/0034-a-person-may-reach-the-mesh.sql @@ -0,0 +1,19 @@ +-- A person who may call the mesh's tools from a workstation. +-- +-- novox/hq design 25 §7. Everything else that reaches the bus is a machine or a module running on +-- one; this is the exception the mesh has always had informally — somebody at a terminal — and never +-- recorded. Until now "the operator" meant whoever held the keys, which is a role and not a record, +-- so nothing could say who may call what. +-- +-- **The authority is a list of tools and nothing else.** A person is not a module: they hold no seat, +-- nothing is addressed to them, nothing is delivered to them, and they have no consumer to +-- acknowledge. What they have is permission to ask. That is why there is no scope column and no node +-- column — a person is not on a machine. +create table person ( + name text primary key, + -- The tools this person may invoke, each `.`, or the single entry `*` for an + -- administrator. Stored as given: the permission is derived from it at every composition, so a + -- normalised form here would be a second opinion about authority (novox/hq ADR 0043). + invokes text[] not null default '{}', + created timestamptz not null default now() +); diff --git a/internal/inventory/people_test.go b/internal/inventory/people_test.go new file mode 100644 index 0000000..1085d57 --- /dev/null +++ b/internal/inventory/people_test.go @@ -0,0 +1,120 @@ +package inventory + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +// Somebody who may call the mesh's tools, and what the bus makes of them. + +// A person's authority is a list of tools, and it becomes exactly that on the bus — nothing on +// control, nothing on nodes, nothing they could publish as a module. +func TestAPersonMayCallToolsAndNothingElse(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + if err := inv.RecordPerson(ctx, Person{Name: "ada", + Invokes: []string{"mesh-catalog.catalog_tools"}}); err != nil { + t.Fatal(err) + } + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if got := records.People["ada"]; len(got) != 1 || got[0] != "mesh-catalog.catalog_tools" { + t.Fatalf("ada may call %v", got) + } + + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var found bool + for _, u := range users { + if u.Username() != "person.ada" { + continue + } + found = true + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.mod.mesh-catalog.tool.catalog_tools" { + t.Errorf("ada may publish %v, which should be the one tool and nothing else", perms.Publish) + } + for _, s := range perms.Publish { + if strings.HasPrefix(s, "mesh.control") || strings.HasPrefix(s, "mesh.node") || + strings.Contains(s, ".event.") { + t.Errorf("a person may publish %s — an event would let them claim a module said "+ + "something, and control is not theirs", s) + } + } + if perms.AllowResponses { + t.Error("a person may answer a request, which is impersonating a module on a bus where " + + "anyone may serve a tool") + } + } + if !found { + t.Fatal("no bus user was derived for a recorded person") + } +} + +// Stating what somebody may call replaces what was there. A list that could only grow is a permission +// nobody can take back. +func TestChangingWhatAPersonMayCallRemovesWhatIsNotNamed(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one", "b.two"}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one"}}); err != nil { + t.Fatal(err) + } + people, err := inv.People(ctx) + if err != nil { + t.Fatal(err) + } + if len(people) != 1 || len(people[0].Invokes) != 1 || people[0].Invokes[0] != "a.one" { + t.Fatalf("ada may call %v; the removed tool is still there", people) + } +} + +// Somebody who may call nothing is refused: there is no reason for them to reach the mesh, and an +// empty list is more likely a mistake than an intention. +func TestSomebodyWhoMayCallNothingIsRefused(t *testing.T) { + inv := ForTest(t) + if err := inv.RecordPerson(context.Background(), Person{Name: "ada"}); err == nil { + t.Fatal("somebody who may call nothing was recorded") + } +} + +// Forgetting somebody takes their credential with them. **Both, or it is not a revocation**: a +// person's row gone and their bus user left behind is a credential that still works and that nothing +// derives. +func TestForgettingAPersonTakesTheirCredential(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one"}}); err != nil { + t.Fatal(err) + } + if _, err := inv.MintBusPassword(ctx, BusUser{Username: "person.ada", Kind: BusPerson}); err != nil { + t.Fatal(err) + } + + if err := inv.ForgetPerson(ctx, "ada"); err != nil { + t.Fatal(err) + } + if _, known, err := inv.BusUserHash(ctx, "person.ada"); err != nil || known { + t.Fatalf("a forgotten person's credential still works: %v %v", known, err) + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := records.People["ada"]; still { + t.Fatal("a forgotten person is still composed into the bus") + } +} -- 2.54.0 From cb77f35a279029a2866ccb0c7c9b9e99b1111703 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 17:22:30 +0200 Subject: [PATCH 36/39] A module may watch a role's events, and the catch-up turns out to be unnecessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving the build outcome onto its role broke the one module that consumes it, and my own agreement check passed anyway. The catalogue's subscription derived `mesh.mod.mesh-build-machine.event.built` — a module namespace for a role's event, which no such module owns — so it started, connected, and its graph stayed empty. The check compared names, and the names agreed: the build machine does emit `built`. Only the subjects disagreed, and a subscription that matches nothing is silence. A consumed name is a module's event unless it names a role, and this package cannot tell by looking — so whoever resolved the declaration says which, the way it already does for a seat held or used. A module that watches a role gets the role's event subject and a consumer filtered on it; watching grants subscribe and nothing else, because hearing what a role announced is not taking part in it. The check now compares the two halves that actually have to match — the subject a consumer subscribes against the subject an emitter publishes — with a case pinning that it catches this exact confusion. Comparing names was checking the easy half. **And that answered the open question about catch-up: there is nothing to build.** The mechanism exists because a queue on the old bus receives only what is published after it is bound, so everything built before the catalogue existed was announced to nobody. A stream is a log and a consumer is a position in it: a consumer created afterwards starts at the beginning, so the builds are simply there. Asked of a real server, since the whole decision rested on it — three builds published with nothing listening, then a consumer created, and all three waiting for it. --- internal/broker/agreement_catalogue_test.go | 112 ++++++++++++++++++++ internal/broker/derived.go | 11 +- internal/broker/nats.go | 17 +++ internal/broker/raise_live_test.go | 51 +++++++++ internal/broker/users.go | 4 +- internal/inventory/busrecords.go | 20 +++- 6 files changed, 211 insertions(+), 4 deletions(-) diff --git a/internal/broker/agreement_catalogue_test.go b/internal/broker/agreement_catalogue_test.go index 7b9e0ec..9a61b3d 100644 --- a/internal/broker/agreement_catalogue_test.go +++ b/internal/broker/agreement_catalogue_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "sort" "strings" "testing" @@ -122,3 +123,114 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat } return emitters, consumers, seats } + +// **Do the derived subjects meet, not just the names?** +// +// The check above compares what a consumer asks for against what an emitter says it emits, by name. It +// passed while the catalogue's subscription pointed at `mesh.mod.mesh-build-machine.event.built` — a +// module namespace for a role's event, which no emitter owns. The names agreed; the subjects did not, +// and the graph stayed empty. +// +// So this compares the thing that actually has to match: the subject a consumer subscribes against the +// subject an emitter publishes. It is the last place the two halves can be held together, because +// after this the server is the only thing that knows and it says nothing — a subscription that matches +// nothing is silence. +func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) { + emitters, consumers, seats := theCataloguesEvents(t) + + // Every subject something publishes: a module's own events, and the events of every role. + published := map[string]bool{} + for _, e := range emitters { + for _, name := range e.Emits { + published["mesh.mod."+e.Module+".event."+name] = true + } + } + for _, s := range seats { + for _, name := range s.Emits { + published["mesh.seat."+s.Name+".event."+name] = true + } + } + + byName := map[string]DeclaredSeat{} + for _, s := range seats { + byName[s.Name] = s + } + + var lonely []string + for _, c := range consumers { + principal := Principal{Kind: KindModule, Node: "one", Module: c.Module, PasswordHash: "x"} + for _, want := range c.Consumes { + emitter, event, named := strings.Cut(want, ".") + if named { + if s, isASeat := byName[emitter]; isASeat { + principal.Watches = append(principal.Watches, + Seat{Name: s.Name, Emits: []string{event}}) + continue + } + } + principal.Consumes = append(principal.Consumes, want) + } + perms, err := PermissionsFor(principal) + if err != nil { + t.Fatalf("%s: %v", c.Module, err) + } + for _, subject := range perms.Subscribe { + if !strings.Contains(subject, ".event.") { + continue + } + if reaches(subject, published) { + continue + } + // A wildcard over emitters reaches whatever arrives later, and an emitter that is not + // installed is ordinary — both are already excused by the check above, so only a subject + // that can never match anything gets here. + if strings.Contains(subject, "*") || strings.Contains(subject, ">") { + continue + } + lonely = append(lonely, c.Module+" subscribes "+subject+", which nothing publishes") + } + } + if len(lonely) > 0 { + sort.Strings(lonely) + t.Fatalf("%d subscription(s) derive to a subject no emitter owns:\n %s", + len(lonely), strings.Join(lonely, "\n ")) + } +} + +// And it catches the thing it exists for: a role's event read as a module's. +func TestTheDerivedSubjectCheckCatchesARolesEventReadAsAModules(t *testing.T) { + published := map[string]bool{"mesh.seat.mesh-build-machine.event.built": true} + // What the derivation produced before a consumed seat name was resolved as one. + if reaches("mesh.mod.mesh-build-machine.event.built", published) { + t.Fatal("a module namespace was treated as reaching a role's event, which is the bug") + } + // And the corrected one does reach it. + if !reaches("mesh.seat.mesh-build-machine.event.built", published) { + t.Fatal("the role's own subject does not reach the role's event") + } +} + +// reaches says whether a subscribed subject admits any published one. +func reaches(subject string, published map[string]bool) bool { + for p := range published { + if admitsSubject(strings.Split(subject, "."), strings.Split(p, ".")) { + return true + } + } + return false +} + +func admitsSubject(pattern, subject []string) bool { + for i, token := range pattern { + if token == ">" { + return i < len(subject) + } + if i >= len(subject) { + return false + } + if token != "*" && token != subject[i] { + return false + } + } + return len(pattern) == len(subject) +} diff --git a/internal/broker/derived.go b/internal/broker/derived.go index b4b0cfa..f5f514a 100644 --- a/internal/broker/derived.go +++ b/internal/broker/derived.go @@ -3,6 +3,7 @@ package broker import ( "fmt" "sort" + "strings" ) // Streams and consumers derived from what modules declare. @@ -103,16 +104,22 @@ type DeclaredSeat struct { // from its name: a module with a consumer per event would need an ack permission per consumer, // and the permission list would stop being derivable from the declaration. func ConsumerFor(p Principal) (Consumer, bool) { - if p.Kind != KindModule || len(p.Consumes) == 0 { + // A module that reacts to anything — a module's events or a role's (novox/hq ADR 0121). Watching + // a role was missing here, so the one module that does it got no consumer at all: it started, + // connected, and its graph stayed empty with nothing anywhere reporting why. + if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0) { return Consumer{}, false } perms, err := PermissionsFor(p) if err != nil { return Consumer{}, false } + // Events, wherever they live: a module's own namespace, and the namespace of any role it watches + // (novox/hq ADR 0121). Tool subjects and inboxes are subscribed directly and are not a consumer's + // business, which is why this is a filter and not the whole list. var filters []string for _, s := range perms.Subscribe { - if len(s) > 9 && s[:9] == "mesh.mod." { + if strings.Contains(s, ".event.") { filters = append(filters, s) } } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 295f2e4..3c18343 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -60,6 +60,15 @@ type Principal struct { Holds []Seat Uses []Seat + // Watches are seats whose events this principal consumes. Separate from Consumes because a + // role's event lives under the seat's namespace and not a module's, and this package cannot tell + // a seat's name from a module's by looking at it — whoever resolved the declaration can, and + // does (novox/hq ADR 0121). + // + // **Found by a consumer reading nothing.** The catalogue consumes the build machine's outcome; + // with that name read as a module's, its subscription pointed at `mesh.mod.mesh-build-machine.…`, + // a namespace no such module owns. Every service started and the graph stayed empty. + Watches []Seat // Invokes are the tools a person may call, as `.`; a single `*` is every tool, // for an administrator. Only meaningful for KindPerson. @@ -260,6 +269,14 @@ func PermissionsFor(p Principal) (Permissions, error) { sub = append(sub, subject) } + // 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a + // role is hearing what it announced, not taking part in it. + for _, w := range p.Watches { + for _, e := range w.Emits { + sub = append(sub, seatSubject(w, "event", e)) + } + } + // 3. Seats it holds: full participation. for _, s := range p.Holds { for _, a := range s.Accepts { diff --git a/internal/broker/raise_live_test.go b/internal/broker/raise_live_test.go index 6281679..202babc 100644 --- a/internal/broker/raise_live_test.go +++ b/internal/broker/raise_live_test.go @@ -143,3 +143,54 @@ func TestRaisingAMeshRolesWorkQueue(t *testing.T) { t.Fatalf("the holder got no worker on the role's queue: %v", err) } } + +// **A consumer created after the fact still sees what came before it**, which is why the mesh needs no +// catch-up at all on this bus (novox/hq 04-ISSUES/050). +// +// On the bus the mesh runs on today a queue receives only what is published after it is bound, so +// everything built before the catalogue existed was announced to nobody — and on a fresh mesh that is +// always the foundation, because those are the things the catalogue needed in order to exist. A whole +// mechanism was built for it: the catalogue asks, the controller re-publishes. +// +// A stream is a log and a consumer is a position in it. A consumer created later starts at the +// beginning by default, so the builds are simply there. Asked of a real server rather than assumed, +// because the whole decision about whether to keep that mechanism rests on it. +func TestAConsumerCreatedAfterwardsStillSeesWhatCameBefore(t *testing.T) { + js := aLiveBus(t) + if err := AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.Context().PurgeStream("EVENTS"); err != nil { + t.Fatal(err) + } + + // Genesis: things are built before anything is listening. + built := []string{"base", "store", "mesh-catalog"} + for _, m := range built { + if _, err := js.Context().Publish("mesh.seat.mesh-build-machine.event.built", + []byte(`{"module":"`+m+`"}`)); err != nil { + t.Fatal(err) + } + } + + // Now the catalogue is installed and the controller creates its consumer. + c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-catalog", + Watches: []Seat{{Name: "mesh-build-machine", Emits: []string{"built"}}}, PasswordHash: "x"}) + if !ok { + t.Fatal("a module that watches a role got no consumer") + } + t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) }) + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + + info, err := js.Context().ConsumerInfo(c.Stream, c.Name) + if err != nil { + t.Fatal(err) + } + if info.NumPending != uint64(len(built)) { + t.Fatalf("a consumer created after %d builds has %d waiting for it — if this is 0 the mesh "+ + "does need a catch-up after all, and the reasoning for deleting it is wrong", + len(built), info.NumPending) + } +} diff --git a/internal/broker/users.go b/internal/broker/users.go index f6fc289..17d0ce4 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -29,6 +29,8 @@ type Declared struct { Holds []Seat // Uses are the seats this module sends to. Uses []Seat + // Watches are the seats whose events it consumes. + Watches []Seat } // Records is what composing a user list needs to know about the mesh, and nothing more. @@ -59,7 +61,7 @@ func Users(r Records) ([]Principal, error) { out = append(out, Principal{ Kind: KindModule, Node: node, Module: d.Module, Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves, - Holds: d.Holds, Uses: d.Uses, + Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, }) } } diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index 4b29bef..a30e624 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -3,6 +3,7 @@ package inventory import ( "context" "fmt" + "strings" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" @@ -93,10 +94,27 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) { // declaredFor is one module's manifest as the composer needs it: what it says about itself, and the // protocol of every seat it holds or uses. func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) broker.Declared { + // A consumed name is a module's event unless it names a seat, and only somebody holding the seat + // set can tell (novox/hq ADR 0121). Split here, because the composer cannot look at a name and + // know — and a role's event read as a module's is a subscription to a namespace nobody owns. + var fromModules []string + var watches []broker.Seat + for _, c := range m.Consumes { + emitter, event, named := strings.Cut(c, ".") + if named { + if s, isASeat := seats[emitter]; isASeat { + watches = append(watches, broker.Seat{Name: s.Name, Emits: []string{event}}) + continue + } + } + fromModules = append(fromModules, c) + } + d := broker.Declared{ Module: m.Module, Emits: m.Emits, - Consumes: m.Consumes, + Consumes: fromModules, + Watches: watches, // The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the // facts a consumer needs to reach a provision, a different meaning under a similar word. Serves: m.Tools, -- 2.54.0 From e5007a7daa3d9af7570d52a6137d89cf5b81bcc1 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 17:33:52 +0200 Subject: [PATCH 37/39] The user list is composed before anything moves onto the bus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by reading the live mesh's own notes before touching it, which is where this was heading next. Composing the bus's user list was gated on the controller already being on the new bus. That cannot work: the server needs its user list *before* anything moves onto it. Step 2 of the whole change is exactly that — the server stands in the mesh carrying nothing, on its own ports, while every node stays where it is. Under the old gating that step was impossible: the module would come up, find no accounts file, and its entrypoint would wait for one the controller had decided not to write. So the only question is whether this machine runs the module that asked for the file. A mesh that never moves has written a user list nothing reads, costing a few hundred bytes on one node. The reverse cost a step that could not be taken. Pinned by a test over the records of a mesh mid-change: everything running, nothing on the new bus, and a user list that contains the controller — because a file without it is a bus its own writer cannot connect to. --- cmd/mesh-controller/plan.go | 20 +++++++++++------- internal/broker/users_test.go | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 017f5b8..8877ccf 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -1160,13 +1160,19 @@ func portsOn( func composeBusUsers(ctx context.Context, inv *inventory.Inventory, onThisNode []catalogue.Manifest) (string, error) { - _, onNATS, err := broker.OnNATS() - if err != nil || !onNATS { - return "", err - } - // Only for the machine holding the bus. Asked of what this push resolves to rather than of the - // seat's holder mesh-wide: the file is a resource of that module, so the question is whether it - // is here. + // **Not gated on which bus the controller is on, and that was a bug.** It read "compose this only + // once the mesh is on the new bus" — which cannot work, because the server needs its user list + // *before* anything moves onto it. Step 2 of the change is exactly that: the server stands in the + // mesh carrying nothing, on its own ports, while every node is still on the old bus (novox/hq + // ADR 0116). Under the old gating that step could not happen: the module would come up, find no + // accounts file, and its entrypoint would wait for one the controller had decided not to write. + // + // So the question is only whether this machine runs the module that asked for the file. A mesh + // that never moves has written a user list nothing reads, which costs a few hundred bytes on one + // node; the reverse cost a step that cannot be taken. + // + // Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a + // resource of that module, so the question is whether it is here. holdsTheBus := false for _, m := range onThisNode { if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { diff --git a/internal/broker/users_test.go b/internal/broker/users_test.go index 8f40e1b..0e21eaf 100644 --- a/internal/broker/users_test.go +++ b/internal/broker/users_test.go @@ -201,3 +201,41 @@ func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) { t.Fatal("a user with no password hash was written") } } + +// **A user list is composed for a bus the mesh has not moved onto yet**, and that is the whole of +// step 2 (novox/hq ADR 0116): the server stands in the mesh carrying nothing, on its own ports, while +// every node is still on the bus it was on. +// +// Pinned because the first version of the composing step got it backwards — it wrote the list only +// once the controller was already on the new bus, which is a step that cannot be taken: the module +// comes up, finds no accounts file, and waits for one the controller had decided not to write. +func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) { + // Exactly the records of a mesh mid-change: everything running, nothing on the new bus. + users, err := Users(Records{ + Nodes: []string{"anchor"}, + Assigned: map[string][]Declared{"anchor": {{Module: "nats"}}}, + }) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for _, u := range users { + hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22) + } + filled, missing := WithPasswords(users, hashes) + if len(missing) != 0 { + t.Fatalf("users with no credential: %v", missing) + } + accounts, err := ComposeAccounts(filled) + if err != nil { + t.Fatal(err) + } + // The controller's own user above all: a file without it is a bus its writer cannot connect to, + // which is what the server would be left holding the moment it starts. + if !strings.Contains(accounts, `user: "controller"`) { + t.Fatalf("the composed list does not contain the controller:\n%s", accounts) + } + if !strings.Contains(accounts, `user: "node.anchor"`) { + t.Errorf("the composed list does not contain the machine running the bus") + } +} -- 2.54.0 From 5fcde512bcc73aeae11cc51663a1738f6d2be12d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 17:39:39 +0200 Subject: [PATCH 38/39] A build is announced under both names on the old bus, or merging breaks the live mesh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by asking what merging this would do to the mesh that is actually running — the only place the question could have been asked, because the tests were green and both buses were self-consistent. Moving the build outcome to the role means a catalogue built from the current manifests listens for the role's name. The catalogue *already running* listens for the module's, because that is what it was told when it was installed. The two do not meet, so merging as it stood would have stopped the live mesh's module graph being updated — silently, since a binding that matches nothing is not an error. A rename on a live bus needs the publisher and the subscriber to change together, and a deployment cannot promise which arrives first. So the old bus announces under both names and the order stops mattering. The module's own name retires with the bus, in step 5's list; nothing has ever run on the bus being built, so there is no legacy name there and this doubling has no counterpart. --- internal/link/builds.go | 8 +++++++ internal/link/builds_current.go | 22 +++++++++++++++-- internal/link/builds_current_test.go | 36 ++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 internal/link/builds_current_test.go diff --git a/internal/link/builds.go b/internal/link/builds.go index 0542f61..8963bc2 100644 --- a/internal/link/builds.go +++ b/internal/link/builds.go @@ -27,6 +27,14 @@ const TheBuildMachine = "mesh-build-machine" func BuildWork() string { return "mesh.seat." + TheBuildMachine + ".accept.build" } func BuildOutcome() string { return "mesh.seat." + TheBuildMachine + ".event.built" } +// KeyRoleBuilt is the build outcome under the role's name, on the bus the mesh runs on today. +// +// The same event as KeyModuleBuilt and published beside it, because a catalogue installed before this +// change listens for the module's name and one installed after listens for the role's. Both, until +// this bus retires: a rename needs publisher and subscriber to change together, and a deployment +// cannot promise which arrives first. +const KeyRoleBuilt = "built" + // Builders is how work reaches a build machine and how the outcome comes back. type Builders interface { // Submit asks for one build and waits for its outcome. diff --git a/internal/link/builds_current.go b/internal/link/builds_current.go index 1f60d41..57165d2 100644 --- a/internal/link/builds_current.go +++ b/internal/link/builds_current.go @@ -161,8 +161,26 @@ func (b *currentBuild) Announce(ctx context.Context, result BuildResult) error { if result.Failed != "" || result.Commit == "" { return nil } - return EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyModuleBuilt, "builder", b.on, - announcementOf(result)) + + // **Announced under both names on this bus, for exactly as long as this bus lives.** + // + // A build's outcome belongs to the role now (novox/hq ADR 0121), so a catalogue built from the + // current manifests listens for the role's name. A catalogue that is *already running* listens for + // the module's, because that is what it was told when it was installed. A rename on a live bus + // needs the publisher and the subscriber to change together, and a merge cannot promise that: one + // of them is deployed first, and in that window the graph silently stops being updated — which is + // the failure this whole change was cleaning up after. + // + // So both, and the order stops mattering. The module's own name goes with the bus, in step 5's + // retirement list; nothing has ever run on the bus being built, so there is no legacy name there + // and this doubling has no counterpart. + announced := announcementOf(result) + if err := EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyModuleBuilt, "builder", b.on, + announced); err != nil { + return err + } + return EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyRoleBuilt, TheBuildMachine, b.on, + announced) } func (b *currentBuild) Done() error { return b.delivery.Ack(false) } diff --git a/internal/link/builds_current_test.go b/internal/link/builds_current_test.go new file mode 100644 index 0000000..7f43f2c --- /dev/null +++ b/internal/link/builds_current_test.go @@ -0,0 +1,36 @@ +package link + +import ( + "strings" + "testing" +) + +// **The old bus announces a build under both names, and that is not belt-and-braces.** +// +// A build's outcome belongs to the role now, so a catalogue built from the current manifests listens +// for the role's name — and a catalogue already running listens for the module's, because that is what +// it was told when it was installed. A rename on a live bus needs publisher and subscriber to change +// together, which a deployment cannot promise: one arrives first, and in that window the module graph +// silently stops being updated. +// +// Caught by asking what merging this would do to the mesh that is actually running, which is the only +// place the question could have been asked — the tests were green and both buses were self-consistent. +func TestTheOldBusAnnouncesABuildUnderBothNames(t *testing.T) { + // The routing key a catalogue installed before the change is bound to. + if KeyModuleBuilt != "module.builder.built" { + t.Fatalf("the module's own name is %q; a catalogue already running is bound to the old one", + KeyModuleBuilt) + } + // And the local name a catalogue built from the current manifests declares, which the old bus's + // client turns into `module..built`. + if KeyRoleBuilt != "built" { + t.Fatalf("the role's event is %q, and a holder emits its verbs bare", KeyRoleBuilt) + } + if TheBuildMachine != "mesh-build-machine" { + t.Fatalf("the role is %q", TheBuildMachine) + } + // The two must differ, or one publish would serve both and this doubling would be pointless. + if strings.HasSuffix(KeyModuleBuilt, "."+TheBuildMachine+"."+KeyRoleBuilt) { + t.Fatal("the two names are the same, so nothing was renamed and this is dead weight") + } +} -- 2.54.0 From dded086b548b86354f1bc6b875c08964a187427c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 27 Sep 2026 17:59:01 +0200 Subject: [PATCH 39/39] `rollout check`: whether this mesh could move its bus, and what is missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rollout moves every node at once, so there is nothing to inspect afterwards and no half to roll back — either the mesh was ready or it was not. That makes the readiness question the valuable half: it costs nothing, it can be asked of a mesh that is serving as many times as you like, and every answer is a thing somebody can go and fix. It reads from records and dials once. Is a bus answering, does a machine hold the seat, has that machine been sent the composed user list, does every machine have a credential for the new bus, does every module that speaks. Each missing thing names its own next step, because "not ready" that cannot be acted on is not an answer — and this is read at the point where the next step is irreversible. **A machine with no credential is the one that must stop it.** It keeps running and cannot come back, and afterwards there is no bus to tell it anything over, so the remedy has to happen first. The message says so. A module that never reaches the bus is not counted as missing a credential. A third of the catalogue never speaks, and listing those would bury the ones that matter. `rollout --confirm` refuses and says why: the move is not being written before its check has been run against a real mesh. And the plan it prints says the old broker stays — it remains an ordinary provider of `amqp` for whatever else uses it, which on this installation is a whole automation layer that has nothing to do with the mesh. This move is not its retirement, and that is why it is survivable: what breaks if it goes wrong is the mesh's ability to change things, not the services its modules serve. --- cmd/mesh-controller/main.go | 2 + cmd/mesh-controller/rollout.go | 193 ++++++++++++++++++++++++++++ cmd/mesh-controller/rollout_test.go | 93 ++++++++++++++ internal/broker/onnats.go | 13 ++ internal/broker/readiness.go | 142 ++++++++++++++++++++ internal/broker/readiness_test.go | 97 ++++++++++++++ 6 files changed, 540 insertions(+) create mode 100644 cmd/mesh-controller/rollout.go create mode 100644 cmd/mesh-controller/rollout_test.go create mode 100644 internal/broker/readiness.go create mode 100644 internal/broker/readiness_test.go diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 3bcf2c1..0727b34 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -114,6 +114,8 @@ func run() error { return planCommand(ctx, args[1:]) case "push": return pushCommand(ctx, args[1:]) + case "rollout": + return rolloutCommand(ctx, args[1:]) case "seats": return seatsCommand(ctx, args[1:]) case "status": diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go new file mode 100644 index 0000000..4e4eb82 --- /dev/null +++ b/cmd/mesh-controller/rollout.go @@ -0,0 +1,193 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5). +// +// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both +// transports and every one of them chooses by a single fact; this is the step that flips it. That +// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays +// bounded until here — and it means the useful work is almost all in the checking. +// +// So `rollout check` is the command that matters and the one that can be run any number of times +// against a mesh that is serving. It answers from records: what is missing, and what would happen. +// `rollout` itself refuses unless the check is clean. +// +// **The old broker is not switched off by this.** It stays an ordinary provider of `amqp` for whatever +// else uses it — on this installation, a whole automation layer that has nothing to do with the mesh +// ([ADR 0119](../../02-DECISIONS/0119-amqp-is-a-provision-not-the-bus.md)). Only the mesh's own +// traffic moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's +// ability to change things, not the services its modules are serving. + +const rolloutUsage = "rollout check | rollout --confirm" + +func rolloutCommand(ctx context.Context, args []string) error { + switch { + case len(args) == 1 && args[0] == "check": + return rolloutCheck(ctx) + case len(args) == 1 && args[0] == "--confirm": + return errors.New( + "the rollout itself is not built yet: `rollout check` answers whether it could run, and " + + "what is missing. Moving every node at once is the one step with nothing to inspect " + + "afterwards, so it is not being written before the check it depends on has been run " + + "against a real mesh") + default: + return errors.New(rolloutUsage) + } +} + +// rolloutCheck says whether the mesh could move, and what would happen if it did. +func rolloutCheck(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + state, err := readinessOf(ctx, inv) + if err != nil { + return err + } + + fmt.Println("the bus this mesh would move to") + if state.TheBus == "" { + fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar) + } else { + standing := "not answering" + if state.ServerStanding { + standing = "answering" + } + fmt.Printf(" %s — %s\n", state.TheBus, standing) + } + fmt.Println() + + fmt.Println("what would move") + for _, step := range broker.WhatMoves(state) { + fmt.Printf(" %s\n", step) + } + fmt.Println() + + why := notReadyOf(state) + if len(why) == 0 { + fmt.Println("nothing is missing: this mesh could move its bus.") + fmt.Println() + fmt.Println("Read `what would move` above once more before running it. Every node moves at the") + fmt.Println("same moment and there is no half-moved state to look at afterwards.") + return nil + } + fmt.Printf("not ready — %d thing(s) to do first:\n", len(why)) + for i, w := range why { + fmt.Printf(" %d. %s\n", i+1, w) + } + return nil +} + +// readinessOf gathers what the mesh knows about its own ability to move. +// +// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the +// point: the answer is only useful if it can be had without committing to anything. +func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) { + state := broker.Readiness{ + Credentialled: map[string]bool{}, + ModuleCredentialled: map[string]bool{}, + // The old broker keeps its other clients on this installation, and saying so is how the plan + // stops reading as a retirement. + OldBusHasOtherClients: true, + } + + address, _, err := broker.OnNATS() + if err != nil { + return state, err + } + state.TheBus = address + if address != "" { + // One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about + // to move onto a server that is not there should hear it here rather than afterwards. + if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil { + state.ServerStanding = true + conn.Close() + } + } + + nodes, err := inv.Nodes(ctx) + if err != nil { + return state, err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return state, err + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return state, err + } + + for _, n := range nodes { + state.Nodes = append(state.Nodes, n.Name) + _, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()] + state.Credentialled[n.Name] = has + + assigned, err := inv.Assigned(ctx, n.Name) + if err != nil { + return state, err + } + for _, module := range assigned { + m, known := shelf[module] + if !known { + continue + } + // The machine that holds the bus seat is the one that would be sent the user list. + if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { + state.Holder = n.Name + state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name) + } + // A module that never speaks needs no credential, so it is not counted as missing one. + if !speaksOnTheBus(m) { + continue + } + named := n.Name + "/" + module + state.Modules = append(state.Modules, named) + _, hasOne := kept[broker.Principal{ + Kind: broker.KindModule, Node: n.Name, Module: module, + }.Username()] + state.ModuleCredentialled[named] = hasOne + } + } + return state, nil +} + +// speaksOnTheBus says whether a module reaches the bus at all. +// +// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential +// would bury the ones that matter under a list nobody can act on. +func speaksOnTheBus(m catalogue.Manifest) bool { + return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 || + len(m.Seats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0 +} + +// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user +// list became part of one. +// +// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away +// has still been sent it, and this question is about whether the mesh did its part. +func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool { + digest, err := inv.Outstanding(ctx, node) + return err == nil && strings.TrimSpace(digest) != "" +} + +// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's +// printing. The reasoning itself is the broker package's, where it is pure. +func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) } diff --git a/cmd/mesh-controller/rollout_test.go b/cmd/mesh-controller/rollout_test.go new file mode 100644 index 0000000..d5a1264 --- /dev/null +++ b/cmd/mesh-controller/rollout_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Whether a mesh could move its bus, read from a real store. +// +// The readiness reasoning has its own tests; this is about the gathering — that the question is +// answered from what the mesh actually holds, on a store with machines and modules in it, without +// writing anything. + +func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) { + inv := inventory.ForTest(t) + ctx := context.Background() + + // A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a + // module that never does. + for _, m := range []catalogue.Manifest{ + {Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf", + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}, + {Module: "gitea", Version: "1", Tools: []string{"repo_create"}}, + {Module: "wallpaper", Version: "1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} { + if err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + + state, err := readinessOf(ctx, inv) + if err != nil { + t.Fatal(err) + } + + if state.Holder != "anchor" { + t.Errorf("the machine holding the bus reads as %q", state.Holder) + } + if len(state.Nodes) != 2 { + t.Errorf("machines read as %v", state.Nodes) + } + // **A module that never speaks is not counted as missing a credential.** A third of the catalogue + // never reaches the bus, and listing those would bury the ones that matter. + for _, m := range state.Modules { + if strings.HasSuffix(m, "/wallpaper") { + t.Errorf("a module that never speaks was counted: %v", state.Modules) + } + } + if len(state.Modules) != 2 { + t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool", + state.Modules) + } + + // Nothing has been minted, so it is not ready — and it says so about each machine by name. + why := notReadyOf(state) + if len(why) == 0 { + t.Fatal("a mesh where nothing has a credential was reported ready to move") + } + said := strings.Join(why, "\n") + for _, name := range []string{"anchor", "laptop"} { + if !strings.Contains(said, name) { + t.Errorf("the refusal does not name %s: %s", name, said) + } + } + + // Mint for one machine and it drops out of the complaint, which is how somebody works through it. + if _, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop", + }); err != nil { + t.Fatal(err) + } + state, err = readinessOf(ctx, inv) + if err != nil { + t.Fatal(err) + } + if !state.Credentialled["laptop"] { + t.Error("a machine that was minted a credential still reads as having none") + } +} diff --git a/internal/broker/onnats.go b/internal/broker/onnats.go index 12d81ed..245f429 100644 --- a/internal/broker/onnats.go +++ b/internal/broker/onnats.go @@ -55,6 +55,19 @@ func CredentialIn(address string) (user, password, bare string) { return user, password, scheme + address[at+1:] } +// BareAddress is a bus address with any credential stripped, for something that only needs to know +// whether a server is answering there. +func BareAddress(address string) string { + _, _, bare := CredentialIn(address) + if bare == "" { + return address + } + if strings.Contains(bare, "://") { + return bare + } + return "nats://" + bare +} + // MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic. // // **Both clients ship and that is the point; both being live is not.** The rollout moves every node diff --git a/internal/broker/readiness.go b/internal/broker/readiness.go new file mode 100644 index 0000000..a1302d3 --- /dev/null +++ b/internal/broker/readiness.go @@ -0,0 +1,142 @@ +package broker + +import ( + "fmt" + "sort" + "strings" +) + +// Whether a mesh could move its bus, and what is missing if not. +// +// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at +// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to +// roll back: either the mesh was ready or it was not. That makes a readiness question the most +// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and +// every answer is a thing somebody can go and fix. +// +// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to +// anything, so it can be asked on a workstation about a mesh it has never reached. + +// Readiness is what the mesh knows about its own ability to move. +type Readiness struct { + // TheBus is the address the mesh's own traffic would move to, empty when nothing names one. + TheBus string + // ServerStanding is whether a bus is reachable at that address, as somebody checked. + ServerStanding bool + // Holder is the node running the module that holds the bus seat, empty when nothing does. + Holder string + // AccountsComposed is whether that node has been sent the composed user list. + AccountsComposed bool + // Nodes is every machine the mesh knows. + Nodes []string + // Credentialled is which of them has a credential for the new bus. + Credentialled map[string]bool + // Modules is every assigned module, as `/`. + Modules []string + // ModuleCredentialled is which of those has one. + ModuleCredentialled map[string]bool + // StillOnTheOldBus is whether anything of the mesh's own still needs the bus it is leaving — + // which is not a reason to stop, because that broker stays as an ordinary provider of `amqp` + // (ADR 0119). Recorded so nobody reads the move as a retirement. + OldBusHasOtherClients bool +} + +// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them. +// +// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check +// that says "not ready" is a check nobody can act on — and this is read at the point where the next +// step is irreversible. +func NotReady(r Readiness) []string { + var why []string + + if strings.TrimSpace(r.TheBus) == "" { + why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+ + "to the address the new server answers on") + } + if !r.ServerStanding { + why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+ + "the one the mesh is on, carrying nothing — assign the module that holds "+ + "mesh-broker and push the machine that runs it") + } + if r.Holder == "" { + why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+ + "list. Assign the module that claims it") + } else if !r.AccountsComposed { + why = append(why, fmt.Sprintf( + "%s holds mesh-broker and has not been sent the composed user list, so the bus would "+ + "refuse every connection. `push %s`", r.Holder, r.Holder)) + } + + // A node with no credential cannot come back after the move, and a node that cannot come back is + // a machine the mesh has lost until somebody goes to it. + var missing []string + for _, n := range r.Nodes { + if !r.Credentialled[n] { + missing = append(missing, n) + } + } + sort.Strings(missing) + if len(missing) > 0 { + why = append(why, fmt.Sprintf( + "%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+ + "one minted before the move, not after — after, there is no bus to ask over", + len(missing), strings.Join(missing, ", "))) + } + + // A module without one keeps running and stops being reachable, which is a smaller fault and still + // one somebody should choose rather than discover. + var quiet []string + for _, m := range r.Modules { + if !r.ModuleCredentialled[m] { + quiet = append(quiet, m) + } + } + sort.Strings(quiet) + if len(quiet) > 0 { + why = append(why, fmt.Sprintf( + "%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+ + "answering tools and hearing events until it is issued one", + len(quiet), strings.Join(quiet, ", "))) + } + + return why +} + +// WhatMoves is what the rollout would do, in order, for somebody reading before they commit. +// +// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so +// the last useful moment to disagree with it is while reading this. +func WhatMoves(r Readiness) []string { + out := []string{ + fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)), + fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)), + } + nodes := append([]string(nil), r.Nodes...) + sort.Strings(nodes) + for _, n := range nodes { + out = append(out, fmt.Sprintf("move %s, and confirm it reports", n)) + } + if len(r.Modules) > 0 { + out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers", + len(r.Modules))) + } + if r.OldBusHasOtherClients { + out = append(out, "leave the old broker running: it stays an ordinary provider of `amqp` for "+ + "whatever else uses it (ADR 0119), and this move is not its retirement") + } + return out +} + +func holderOr(node string) string { + if node == "" { + return "whichever machine holds mesh-broker" + } + return node +} + +func busOr(address string) string { + if address == "" { + return "the new bus" + } + return address +} diff --git a/internal/broker/readiness_test.go b/internal/broker/readiness_test.go new file mode 100644 index 0000000..827b786 --- /dev/null +++ b/internal/broker/readiness_test.go @@ -0,0 +1,97 @@ +package broker + +import ( + "strings" + "testing" +) + +// Whether a mesh could move its bus. +// +// Every case here is a way of moving that leaves something behind, and the one that matters most is a +// machine with no credential: after the move there is no bus to ask it over, so it is lost until +// somebody walks to it. + +func aMeshReadyToMove() Readiness { + return Readiness{ + TheBus: "nats://127.0.0.1:5671", ServerStanding: true, + Holder: "anchor", AccountsComposed: true, + Nodes: []string{"anchor", "laptop"}, + Credentialled: map[string]bool{"anchor": true, "laptop": true}, + Modules: []string{"anchor/gitea"}, + ModuleCredentialled: map[string]bool{"anchor/gitea": true}, + } +} + +func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) { + if why := NotReady(aMeshReadyToMove()); len(why) != 0 { + t.Fatalf("a mesh with everything in place was refused: %v", why) + } +} + +// **A machine with no credential is the one that must stop this.** It keeps running and cannot come +// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and +// the message says so. +func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) { + r := aMeshReadyToMove() + r.Credentialled = map[string]bool{"anchor": true} + + why := NotReady(r) + if len(why) == 0 { + t.Fatal("a machine that could not come back did not stop the move") + } + said := strings.Join(why, "\n") + if !strings.Contains(said, "laptop") { + t.Errorf("the refusal does not name the machine: %s", said) + } + if !strings.Contains(said, "before the move") { + t.Errorf("the refusal does not say the remedy comes first: %s", said) + } +} + +// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it, +// and each names its own next step, because "not ready" that cannot be acted on is not an answer. +func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) { + for _, c := range []struct { + what string + break_ func(*Readiness) + says string + }{ + {"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar}, + {"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"}, + {"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"}, + {"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"}, + {"a module with none", func(r *Readiness) { + r.ModuleCredentialled = map[string]bool{} + }, "anchor/gitea"}, + } { + r := aMeshReadyToMove() + c.break_(&r) + why := NotReady(r) + if len(why) == 0 { + t.Errorf("%s did not stop the move", c.what) + continue + } + if !strings.Contains(strings.Join(why, "\n"), c.says) { + t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why) + } + } +} + +// What the move would do is written out rather than summarised, because this is the one step with +// nothing to inspect afterwards — so reading it is the last chance to disagree. +func TestWhatMovesNamesEveryMachineAndSaysTheOldBrokerStays(t *testing.T) { + r := aMeshReadyToMove() + r.OldBusHasOtherClients = true + steps := strings.Join(WhatMoves(r), "\n") + + for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} { + if !strings.Contains(steps, want) { + t.Errorf("the plan does not mention %q:\n%s", want, steps) + } + } + // Said explicitly, so nobody reads the move as switching the old broker off — it stays serving + // whatever else uses it, and that is a decision already taken. + if !strings.Contains(steps, "not its retirement") { + t.Errorf("the plan does not say the old broker stays:\n%s", steps) + } +} -- 2.54.0