diff --git a/Makefile b/Makefile index 2e21062..a41e6d7 100644 --- a/Makefile +++ b/Makefile @@ -86,13 +86,19 @@ proxy-image: # The whole gate. Raises a database, runs everything against it, and takes it down again -- # including when the tests fail, which is why the teardown is not conditional. +# +# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on +# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names, +# because those names are the mesh's and a test cannot choose others. Two packages doing that at once +# is one deleting a consumer the other is reading through, and the failure lands in whichever test +# was reading, as "no response from stream". That reads as a bug in the code under test. check: fmt vet postgres - @go test ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status + @go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status # Without a database the live tests skip rather than fail, so this is the honest subset and not -# the gate. +# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not. test: - go test ./... + go test -p 1 ./... vet: go vet ./... diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index c4fa5e1..e27d4b3 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -29,10 +29,10 @@ import ( "os/signal" "strings" "syscall" - "time" amqp "github.com/rabbitmq/amqp091-go" + "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/link" ) @@ -115,72 +115,63 @@ func run() error { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - conn, err := dial(credential) - if err != nil { - // Not quoted back: the URL carries this builder's broker password. - return fmt.Errorf("cannot reach the broker: %w", err) - } - defer conn.Close() - channel, err := conn.Channel() - if err != nil { - return err - } - defer channel.Close() - - if _, err := channel.QueueDeclare(link.BuildQueue, true, false, false, false, nil); err != nil { - return err - } - // One at a time. A build machine that took five requests at once would run five container - // builds against one runtime and finish all of them slower than it would have finished the - // first — and the queue is what shares work between machines, so nothing is lost by it. - if err := channel.Qos(1, 0, false); err != nil { - return err - } - - // Not auto-acknowledged. A request acknowledged on arrival is a build that vanishes if this - // process dies mid-way, with nobody waiting on it ever hearing why. - requests, err := channel.ConsumeWithContext(ctx, link.BuildQueue, "mesh-builder", - false, false, false, false, nil) + machine, err := takeWorkFrom(credential, on) if err != nil { return err } + defer machine.Close() fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry) publisher := builder.Registry{Address: registry, Run: builder.Command} - for { - select { - case <-ctx.Done(): - fmt.Println("stopping") - return nil - case delivery, ok := <-requests: - if !ok { - return fmt.Errorf("the broker closed the connection") - } - answer(ctx, channel, publisher, on, workspace, delivery) - } + return machine.Take(ctx, func(ctx context.Context, work link.Build) { + answer(ctx, publisher, on, workspace, work) + }) +} + +// takeWorkFrom opens this machine's link to whichever bus the mesh is on. +// +// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build +// machine told about both would take work from one and answer on the other, and every log line would +// say it was fine. +func takeWorkFrom(credential Credential, on string) (link.BuildMachine, error) { + address, onNATS, err := broker.OnNATS() + if err != nil { + return nil, err } + if err := broker.MustBeOneBus(credential.URL, address); err != nil { + return nil, err + } + if onNATS { + js, err := broker.Dial(address) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus at %s: %w", address, err) + } + return link.MachineOverNATS(js, on), nil + } + + conn, err := dial(credential) + if err != nil { + // Not quoted back: the URL carries this builder's broker password. + return nil, fmt.Errorf("cannot reach the broker: %w", err) + } + channel, err := conn.Channel() + if err != nil { + conn.Close() + return nil, err + } + return link.MachineOverCurrent(conn, channel, on), nil } // answer does one build and says what happened, whichever way it went. -func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publisher, - on, workspace string, delivery amqp.Delivery) { +func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build) { + request := work.Request() - // **First thing, and to stdout.** A build request that arrives and produces no visible line - // until it either finishes or fails is indistinguishable from one that never arrived — which - // cost a long diagnosis against a running mesh, chasing "the handler never fired" when the - // truth was only that the handler said nothing until the end. - fmt.Fprintf(os.Stderr, "a build request arrived (%d bytes)\n", len(delivery.Body)) - - var request link.BuildRequest - if err := json.Unmarshal(delivery.Body, &request); err != nil { - // Unreadable. Acknowledged and dropped rather than requeued: a message this builder - // cannot parse will not become parseable by being delivered again, and requeueing it - // would put it in front of every real request for ever. - fmt.Fprintf(os.Stderr, "a request could not be read and was dropped: %v\n", err) - _ = delivery.Ack(false) - return - } + // **First thing, and to stdout.** A build request that arrives and produces no visible line until + // it either finishes or fails is indistinguishable from one that never arrived — which cost a long + // diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that + // the handler said nothing until the end. + fmt.Fprintf(os.Stderr, "a build request arrived for %s\n", request.Repository) result := link.BuildResult{ ID: request.ID, Repository: request.Repository, Path: request.Path, @@ -199,8 +190,8 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis var built builder.Result if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a - // build that could not have resolved its dependencies is refused in front of the reason, - // not after a clone that then fails at npm ci. + // build that could not have resolved its dependencies is refused in front of the reason, not + // after a clone that then fails at npm ci. built, err = builder.Build(ctx, builder.Command, publisher, request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, forgeFrom(), @@ -230,67 +221,19 @@ func answer(ctx context.Context, channel *amqp.Channel, publisher builder.Publis } } - body, err := json.Marshal(result) - if err != nil { - fmt.Fprintf(os.Stderr, "cannot report a build: %v\n", err) - _ = delivery.Ack(false) + if err := work.Announce(ctx, result); err != nil { + // Said, not fatal: the build happened. A build reported as failed because announcing it + // failed is a lie about work that was done — and the request stays unsettled below only if + // nothing was said at all, so another machine can try. + fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err) return } - // Always through the exchange, whether or not somebody is waiting. - // - // **Never the default exchange.** Permission there is granted per exchange rather than per - // queue, so a builder allowed to use it could publish into any node's queue — the privilege a - // build machine most obviously should not have. An asker binds its own reply queue to this - // key and filters by correlation; a control plane that records builds is bound to it too, so - // a result nobody asked for is still kept rather than reported into the void. - publishCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - if err := channel.PublishWithContext(publishCtx, link.Exchange, link.KeyBuilt, false, false, - amqp.Publishing{ - ContentType: "application/json", - CorrelationId: result.ID, - Body: body, - }); err != nil { - fmt.Fprintf(os.Stderr, "cannot answer a build request: %v\n", err) + // Settled only once the outcome is away, so a machine that dies before answering leaves the work + // for another rather than losing it. + if err := work.Done(); err != nil { + fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err) } - // **And announced, which is a different act from answering.** The reply goes to whoever asked - // and is correlated to their request; this says to the whole mesh that a module now exists at - // a commit, and the catalogue places it in the module graph (novox/hq ADR 0072). A build - // nobody asked for still has to be announced, or the graph knows less than the registry does. - // - // Only on success: a failed build produced no module-version, and announcing one would put - // something in the graph that was never made. - if result.Failed == "" && result.Commit != "" { - announced := map[string]any{ - "module": moduleOf(result.Manifest), "commit": result.Commit, - "repository": result.Repository, "path": result.Path, "ref": result.Ref, - "manifest": json.RawMessage(result.Manifest), "against": result.Against, - "made": result.Made, - } - if err := link.EmitEvent(publishCtx, channel, link.KeyModuleBuilt, "builder", on, announced); err != nil { - // Said, not fatal: the build happened and was answered. A module the catalogue has not - // heard of is a gap somebody can close; a build reported as failed because announcing - // it failed is a lie about work that was done. - fmt.Fprintf(os.Stderr, " built, but could not announce it: %v\n", err) - } - } - - // Acknowledged only once the answer is away, so a builder that dies before answering leaves - // the request for another machine rather than losing it. - _ = delivery.Ack(false) -} - -// moduleOf reads the module's name out of the manifest it just built, which is the only place it is -// authoritative — the request named a repository and a path, not a module. -func moduleOf(manifest json.RawMessage) string { - var named struct { - Module string `json:"module"` - } - if err := json.Unmarshal(manifest, &named); err != nil { - return "" - } - return named.Module } // packagesFrom is where a build resolves the mesh's own published packages — the SDK above all diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 582b04c..600d36a 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -395,7 +395,13 @@ func buildOne(ctx context.Context, source buildSource, path, ref string, wait ti } fmt.Println() - result, err := link.RequestBuild(ctx, server.Channel(), request, wait) + ask, err := askOver(server) + if err != nil { + return err + } + defer ask.Close() + + result, err := ask.Submit(ctx, request, wait) if err != nil { return err } @@ -472,7 +478,13 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai } defer server.Close() - result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{ + ask, err := askOver(server) + if err != nil { + return err + } + defer ask.Close() + + result, err := ask.Submit(ctx, link.BuildRequest{ ID: fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()), Repository: repository, Path: path, Ref: ref, Held: heldBy(ctx), @@ -557,3 +569,22 @@ func heldBy(ctx context.Context) map[string]string { } return routed } + +// askOver opens the way a build is asked for, on whichever bus the mesh is on. +// +// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus +// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus +// being built it dials, because a build request is a one-shot and holds nothing else. +func askOver(server *link.Server) (link.Builders, error) { + address, onNATS, err := broker.OnNATS() + if err != nil { + return nil, err + } + if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), address); err != nil { + return nil, err + } + if onNATS { + return link.BuildsOverNATS(address) + } + return link.BuildsOverCurrent(server.Channel()), nil +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 74c5a55..99a5cfa 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -114,6 +114,8 @@ func run() error { return planCommand(ctx, args[1:]) case "push": return pushCommand(ctx, args[1:]) + case "rollout": + return rolloutCommand(ctx, args[1:]) case "seats": return seatsCommand(ctx, args[1:]) case "seat": diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 1d0d72d..fdb719f 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -257,6 +257,21 @@ func moduleCommand(ctx context.Context, args []string) error { return err } + // Which bus this mesh is on. A module gets a credential for exactly one, and the two are + // made in entirely different ways: on the bus the mesh runs on today an account is a + // management call, and on the bus being built it is a row the next composition writes into + // the server's user list (novox/hq design 25 §4). + busAddress, onNATS, err := broker.OnNATS() + if err != nil { + return err + } + if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil { + return err + } + if onNATS { + return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress) + } + management, err := broker.ManagementFromEnvironment() if err != nil { return err @@ -567,3 +582,77 @@ func mayIssue(m catalogue.Manifest) error { } return nil } + +// issueOnTheNewBus gives an assigned module its credential on the bus being built. +// +// **Three things differ from a management call, and each is the point of the move.** The credential +// is minted into the mesh's records and becomes usable at the next composition, so there is no +// server to be reachable for this to work. The password travels beside the address rather than inside +// it, because the runtime's contract already separates them and a credential embedded in a URL is one +// that leaks into every log line that prints a connection. And the module's durable consumer is +// derived from what it declared rather than declared by name, so a module cannot ask for delivery of +// something it did not say it consumes. +func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, + node, busAddress string) error { + + user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username() + password, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: user, Kind: inventory.BusModule, Node: node, Module: m.Module, + }) + if err != nil { + return err + } + + // Where the module is told to find the bus, and what certificate it must present. The same pair + // a node is told, for the same reason: a mesh's bus presents its own certificate, in no public + // trust store, so an address alone fails at TLS. + known, err := broker.FromEnvironment() + if err != nil { + return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err) + } + reachable, err := brokerReachableAt(ctx, inv, known, node) + if err != nil { + return err + } + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node"` + Module string `json:"module"` + User string `json:"user"` + Password string `json:"password"` + }{ + URL: "nats://" + reachable, Fingerprint: known.Fingerprint, + Node: node, Module: m.Module, User: user, Password: password, + }) + if err != nil { + return err + } + if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil { + return err + } + + // And how it hears what it consumes. Derived from its declaration, and only when it declared + // something: a module that consumes nothing needs no consumer, and creating one would be a + // durable subscription nobody reads. + if consumer, needed := broker.ConsumerFor(broker.Principal{ + Kind: broker.KindModule, Node: node, Module: m.Module, + Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools, + }); needed { + js, err := broker.Dial(busAddress) + if err != nil { + return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+ + "how %s hears what it consumes: %w", m.Module, err) + } + defer js.Close() + if err := js.EnsureConsumer(consumer); err != nil { + return err + } + } + + fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module) + fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node) + fmt.Printf(" and it works once the bus has been told: the user list is composed into the " + + "machine holding mesh-broker\n") + return nil +} diff --git a/cmd/mesh-controller/operator.go b/cmd/mesh-controller/operator.go index af037eb..0bf944d 100644 --- a/cmd/mesh-controller/operator.go +++ b/cmd/mesh-controller/operator.go @@ -2,12 +2,15 @@ package main import ( "context" + "encoding/json" "errors" "flag" "fmt" "os" "strings" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/secrets" ) @@ -26,9 +29,25 @@ import ( // operator key make [--out ] make a keypair: private half to the file, public half printed // operator key set tell the mesh which key to seal to // operator key show the public key, its fingerprint, and what it can recover -const operatorUsage = "operator key make [--out ] | operator key set [--replace] | operator key show" +const operatorUsage = "operator key make [--out ] | operator key set [--replace] | " + + "operator key show | operator issue --invokes | operator revoke | " + + "operator list" func operatorCommand(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New(operatorUsage) + } + // The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because + // both answer "who, other than a machine, may do something here" — and a person reading this + // command's usage is asking exactly that. + switch args[0] { + case "issue": + return personIssue(ctx, args[1:]) + case "revoke": + return personRevoke(ctx, args[1:]) + case "list": + return personList(ctx) + } if len(args) < 2 || args[0] != "key" { return errors.New(operatorUsage) } @@ -160,3 +179,125 @@ func readPrivateKey(path string) (string, error) { } return strings.TrimSpace(string(raw)), nil } + +// personIssue gives somebody a credential for the mesh's tools, and prints it once. +// +// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the +// credential exists anywhere but on the workstation that will use it — the same contract a token has, +// and for the same reason: a credential recoverable from the mesh's store has the store's blast +// radius. +func personIssue(ctx context.Context, args []string) error { + set := flag.NewFlagSet("operator issue", flag.ContinueOnError) + invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 1 { + return errors.New("operator issue --invokes ") + } + name := set.Arg(0) + if *invokes == "" { + return errors.New( + "say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " + + "or --invokes '*' for an administrator") + } + var tools []string + for _, t := range strings.Split(*invokes, ",") { + if t = strings.TrimSpace(t); t != "" { + tools = append(tools, t) + } + } + + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil { + return err + } + // Refused here rather than at the next composition, where it would stop the whole file being + // written for everybody. A name that cannot be part of a subject is one the server would read as + // a wider permission than anybody granted. + if _, err := broker.PermissionsFor(broker.Principal{ + Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x", + }); err != nil { + return err + } + + user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username() + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson}) + if err != nil { + return err + } + + where, err := broker.FromEnvironment() + if err != nil && !errors.Is(err, broker.ErrNotConfigured) { + return err + } + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + User string `json:"user"` + Password string `json:"password"` + Person string `json:"person"` + Invokes []string `json:"invokes"` + }{ + URL: "nats://" + where.Address, Fingerprint: where.Fingerprint, + User: user, Password: password, Person: name, Invokes: tools, + }) + if err != nil { + return err + } + + fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", ")) + fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash") + fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker") + fmt.Println() + fmt.Println(string(held)) + return nil +} + +func personRevoke(ctx context.Context, args []string) error { + if len(args) != 1 { + return errors.New("operator revoke ") + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + + if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil { + return err + } + // **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops + // working when the file no longer names it. Said plainly, because "revoked" that still works for + // another minute is worth knowing about. + fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+ + "push the machine holding mesh-broker to make it so\n", args[0]) + return nil +} + +func personList(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + + people, err := open.inventory.People(ctx) + if err != nil { + return err + } + if len(people) == 0 { + fmt.Println("nobody but machines reaches this mesh") + return nil + } + for _, p := range people { + fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", ")) + } + return nil +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 85f9a0f..2c00f9b 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -621,13 +621,20 @@ func renderingFor(ctx context.Context, open *stores, node string, if err != nil { return catalogue.Rendering{}, inventory.Node{}, err } + // The bus's user list, for the machine that runs the bus. Composed per push rather than kept, + // because it is a function of the mesh's records and a kept copy could disagree with them. + busUsers, err := composeBusUsers(ctx, inv, plan.Modules) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, - Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation, Kept: kept, - Adopted: record.Adopted, - Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built, + Suffix: overlay.Suffix(), MeshRange: meshRange, Accounts: accounts, Foundation: foundation, + Kept: kept, Adopted: record.Adopted, + Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built, + BusUsers: busUsers, }, record, nil } @@ -1187,6 +1194,76 @@ func portsOn( return out, nil } +// composeBusUsers is the bus's user list, for a push to the machine that runs the bus. +// +// Empty for every other machine, and for every machine while the mesh is on the bus it runs on +// today — where accounts are a management call and there is no file to write. +// +// **Composed on each push, never kept.** The list is a function of the mesh's records (who exists, +// what runs where, what each declares), and a stored copy would be a second account of who may reach +// the bus, able to disagree with the records while both looked internally consistent (ADR 0043). +// +// A user the mesh has never minted a password for is **left out and said**, not written as a user +// without one — the composer refuses that, because a user with no password is a user anybody is. That +// is an ordinary situation with an obvious remedy (`module issue`, or enrolling), so the push carries +// the rest rather than failing: a bus that is missing one module's user is a mesh where that module +// cannot connect, and a bus with no file at all is a mesh where nothing can. +func composeBusUsers(ctx context.Context, inv *inventory.Inventory, + onThisNode []catalogue.Manifest) (string, error) { + + // **Not gated on which bus the controller is on, and that was a bug.** It read "compose this only + // once the mesh is on the new bus" — which cannot work, because the server needs its user list + // *before* anything moves onto it. Step 2 of the change is exactly that: the server stands in the + // mesh carrying nothing, on its own ports, while every node is still on the old bus (novox/hq + // ADR 0116). Under the old gating that step could not happen: the module would come up, find no + // accounts file, and its entrypoint would wait for one the controller had decided not to write. + // + // So the question is only whether this machine runs the module that asked for the file. A mesh + // that never moves has written a user list nothing reads, which costs a few hundred bytes on one + // node; the reverse cost a step that cannot be taken. + // + // Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a + // resource of that module, so the question is whether it is here. + holdsTheBus := false + for _, m := range onThisNode { + if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { + holdsTheBus = true + } + } + if !holdsTheBus { + return "", nil + } + + records, err := inv.BusRecords(ctx) + if err != nil { + return "", err + } + users, err := broker.Users(records) + if err != nil { + return "", err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return "", err + } + hashes := make(map[string]string, len(kept)) + for name, u := range kept { + hashes[name] = u.PasswordHash + } + filled, missing := broker.WithPasswords(users, hashes) + if len(missing) > 0 { + fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+ + "for: %s. Each is a user that cannot connect until one is issued\n", + len(missing), strings.Join(missing, ", ")) + } + if len(filled) == 0 { + return "", fmt.Errorf( + "this machine runs the bus and not one user has a credential, so the composed list " + + "would refuse every connection in the mesh") + } + return broker.ComposeAccounts(filled) +} + // foundationPortsFor is the broker port, kept only when a module resolved onto this node listens // on it (novox/hq issue: the broker opening leaked onto every node). The foundation opening // exists to WIDEN the broker's `from: mesh` port to from-anywhere, because a machine enrolling is diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 74ec025..5ad9161 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -77,12 +77,34 @@ func serve(ctx context.Context) error { "reconnect. Set %s and %s.\n", broker.AddressVar, broker.CertificateVar) } - work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known} + // **Which bus this mesh is on, read once** (novox/hq ADR 0116 step 5). Both clients ship; both + // being live is refused, because a mesh half on each is one where a declaration goes out on one + // and the report comes back on the other, and every component logs success while it happens. + busAddress, onNATS, err := broker.OnNATS() + if err != nil { + return err + } + if err := broker.MustBeOneBus(os.Getenv(broker.AMQPVarName), busAddress); err != nil { + return err + } + + work := link.Enrolment{Inventory: inv, Identity: ident, Management: management, Broker: known, + OnNATS: onNATS} server, err := link.Connect(work, work) if err != nil { return err } defer server.Close() + + // The bus's own objects, asserted on every start. **Not created once at genesis**: a stream + // somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was + // replaced all have records and no objects — and a node whose consumer is missing hears nothing + // while everything else about it looks correct. + if onNATS { + if err := raiseTheBus(ctx, inv, busAddress); err != nil { + return err + } + } // And build results nobody was waiting for. A build triggered any other way than `build` // would otherwise be reported into the void, which is the same as not reporting it. server.Records(builds{inv}) @@ -142,7 +164,7 @@ func declare(ctx context.Context, args []string) error { } defer server.Close() - if err := link.Declare(ctx, server.Channel(), ident, node, raw, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, node, raw, 15*time.Second); err != nil { return err } fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) @@ -310,7 +332,7 @@ func pushCommand(ctx context.Context, args []string) error { if err != nil { return err } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } // After it is away, not before. A digest recorded for something that failed to send would @@ -393,7 +415,7 @@ func pushCommand(ctx context.Context, args []string) error { return declarationWith(held, open, node, plan, settings, gens, Allocating) }, func(s readyNode, body []byte) error { - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } @@ -617,7 +639,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error { if err != nil { return err } - if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + if err := link.Declare(ctx, link.OverCurrent{Channel: server.Channel()}, ident, s.node, body, 15*time.Second); err != nil { return err } record, err := inv.NodeByName(ctx, s.node) @@ -670,3 +692,57 @@ func wouldSend(ctx context.Context, open *stores, } return out, nil } + +// raiseTheBus asserts the streams and consumers the mesh's own traffic needs. +// +// **Every start, and it says what it did.** The objects are the mesh's, created by nothing else — +// the controller is their only writer (design 25 §3) — so a mesh that came up without them is one +// where nodes connect, authenticate, and hear nothing. Said rather than silent for the reason the +// first line of `serve` is said: a log that is quiet on success and loud on failure reads as broken +// when it is working. +func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string) error { + js, err := broker.Dial(address) + if err != nil { + return fmt.Errorf("the mesh is on the bus at %s and this control plane cannot reach it: %w", + address, err) + } + defer js.Close() + + // **Its own user, before anything else.** The controller's account is created by the installer at + // a bootstrap password, before there is a controller to mint one — so nothing recorded a hash for + // it, and the first composition would leave the writer out of the file it was writing. Recorded + // only if absent: a credential the mesh minted since is the one that counts. + // **Its own user, before anything else it does here.** The controller's account is created by the + // installer at a bootstrap password, before there is a controller to mint one — so nothing + // recorded a hash for it, and the first composition would leave the writer out of the file it was + // writing: a bus nothing can connect to, produced by the thing connected to it. Recorded only if + // absent, so a restart cannot put the bootstrap credential back over a rotated one. + if user, password, _ := broker.CredentialIn(address); user != "" && password != "" { + if err := inv.SeedBusUser(ctx, inventory.BusUser{ + Username: user, Kind: inventory.BusController, + }, password); err != nil { + return fmt.Errorf("cannot record the credential this control plane is using: %w", err) + } + } + + nodes, err := inv.Nodes(ctx) + if err != nil { + return err + } + names := make([]string, 0, len(nodes)) + for _, n := range nodes { + names = append(names, n.Name) + } + if err := broker.Raise(js, names); err != nil { + return err + } + // The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder, + // deliberately: work queues until somebody arrives to do it, so assigning a build machine a week + // after something started asking for builds flushes the backlog instead of having lost it. + if err := broker.RaiseSeats(js, inventory.MeshSeats(), nil); err != nil { + return err + } + fmt.Printf("the bus at %s has its streams, and %d machine(s) can hear a declaration\n", + address, len(names)) + return nil +} diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go new file mode 100644 index 0000000..c386490 --- /dev/null +++ b/cmd/mesh-controller/rollout.go @@ -0,0 +1,193 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5). +// +// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both +// transports and every one of them chooses by a single fact; this is the step that flips it. That +// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays +// bounded until here — and it means the useful work is almost all in the checking. +// +// So `rollout check` is the command that matters and the one that can be run any number of times +// against a mesh that is serving. It answers from records: what is missing, and what would happen. +// `rollout` itself refuses unless the check is clean. +// +// **The old broker is not switched off by this.** It stays an ordinary provider of `amqp` for whatever +// else uses it — on this installation, a whole automation layer that has nothing to do with the mesh +// ([ADR 0119](../../02-DECISIONS/0119-amqp-is-a-provision-not-the-bus.md)). Only the mesh's own +// traffic moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's +// ability to change things, not the services its modules are serving. + +const rolloutUsage = "rollout check | rollout --confirm" + +func rolloutCommand(ctx context.Context, args []string) error { + switch { + case len(args) == 1 && args[0] == "check": + return rolloutCheck(ctx) + case len(args) == 1 && args[0] == "--confirm": + return errors.New( + "the rollout itself is not built yet: `rollout check` answers whether it could run, and " + + "what is missing. Moving every node at once is the one step with nothing to inspect " + + "afterwards, so it is not being written before the check it depends on has been run " + + "against a real mesh") + default: + return errors.New(rolloutUsage) + } +} + +// rolloutCheck says whether the mesh could move, and what would happen if it did. +func rolloutCheck(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + state, err := readinessOf(ctx, inv) + if err != nil { + return err + } + + fmt.Println("the bus this mesh would move to") + if state.TheBus == "" { + fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar) + } else { + standing := "not answering" + if state.ServerStanding { + standing = "answering" + } + fmt.Printf(" %s — %s\n", state.TheBus, standing) + } + fmt.Println() + + fmt.Println("what would move") + for _, step := range broker.WhatMoves(state) { + fmt.Printf(" %s\n", step) + } + fmt.Println() + + why := notReadyOf(state) + if len(why) == 0 { + fmt.Println("nothing is missing: this mesh could move its bus.") + fmt.Println() + fmt.Println("Read `what would move` above once more before running it. Every node moves at the") + fmt.Println("same moment and there is no half-moved state to look at afterwards.") + return nil + } + fmt.Printf("not ready — %d thing(s) to do first:\n", len(why)) + for i, w := range why { + fmt.Printf(" %d. %s\n", i+1, w) + } + return nil +} + +// readinessOf gathers what the mesh knows about its own ability to move. +// +// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the +// point: the answer is only useful if it can be had without committing to anything. +func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) { + state := broker.Readiness{ + Credentialled: map[string]bool{}, + ModuleCredentialled: map[string]bool{}, + // The old broker keeps its other clients on this installation, and saying so is how the plan + // stops reading as a retirement. + OldBusHasOtherClients: true, + } + + address, _, err := broker.OnNATS() + if err != nil { + return state, err + } + state.TheBus = address + if address != "" { + // One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about + // to move onto a server that is not there should hear it here rather than afterwards. + if conn, err := nats.Connect(broker.BareAddress(address), nats.Timeout(5*time.Second)); err == nil { + state.ServerStanding = true + conn.Close() + } + } + + nodes, err := inv.Nodes(ctx) + if err != nil { + return state, err + } + kept, err := inv.BusUsers(ctx) + if err != nil { + return state, err + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return state, err + } + + for _, n := range nodes { + state.Nodes = append(state.Nodes, n.Name) + _, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()] + state.Credentialled[n.Name] = has + + assigned, err := inv.Assigned(ctx, n.Name) + if err != nil { + return state, err + } + for _, module := range assigned { + m, known := shelf[module] + if !known { + continue + } + // The machine that holds the bus seat is the one that would be sent the user list. + if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { + state.Holder = n.Name + state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name) + } + // A module that never speaks needs no credential, so it is not counted as missing one. + if !speaksOnTheBus(m) { + continue + } + named := n.Name + "/" + module + state.Modules = append(state.Modules, named) + _, hasOne := kept[broker.Principal{ + Kind: broker.KindModule, Node: n.Name, Module: module, + }.Username()] + state.ModuleCredentialled[named] = hasOne + } + } + return state, nil +} + +// speaksOnTheBus says whether a module reaches the bus at all. +// +// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential +// would bury the ones that matter under a list nobody can act on. +func speaksOnTheBus(m catalogue.Manifest) bool { + return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 || + len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0 +} + +// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user +// list became part of one. +// +// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away +// has still been sent it, and this question is about whether the mesh did its part. +func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool { + digest, err := inv.Outstanding(ctx, node) + return err == nil && strings.TrimSpace(digest) != "" +} + +// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's +// printing. The reasoning itself is the broker package's, where it is pure. +func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) } diff --git a/cmd/mesh-controller/rollout_test.go b/cmd/mesh-controller/rollout_test.go new file mode 100644 index 0000000..ba5ac2f --- /dev/null +++ b/cmd/mesh-controller/rollout_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Whether a mesh could move its bus, read from a real store. +// +// The readiness reasoning has its own tests; this is about the gathering — that the question is +// answered from what the mesh actually holds, on a store with machines and modules in it, without +// writing anything. + +func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) { + inv := inventory.ForTest(t) + ctx := context.Background() + + // A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a + // module that never does. + for _, m := range []catalogue.Manifest{ + {Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf", + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}}, + {Module: "gitea", Version: "1", Tools: []string{"repo_create"}}, + {Module: "wallpaper", Version: "1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} { + if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { + t.Fatal(err) + } + } + + state, err := readinessOf(ctx, inv) + if err != nil { + t.Fatal(err) + } + + if state.Holder != "anchor" { + t.Errorf("the machine holding the bus reads as %q", state.Holder) + } + if len(state.Nodes) != 2 { + t.Errorf("machines read as %v", state.Nodes) + } + // **A module that never speaks is not counted as missing a credential.** A third of the catalogue + // never reaches the bus, and listing those would bury the ones that matter. + for _, m := range state.Modules { + if strings.HasSuffix(m, "/wallpaper") { + t.Errorf("a module that never speaks was counted: %v", state.Modules) + } + } + if len(state.Modules) != 2 { + t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool", + state.Modules) + } + + // Nothing has been minted, so it is not ready — and it says so about each machine by name. + why := notReadyOf(state) + if len(why) == 0 { + t.Fatal("a mesh where nothing has a credential was reported ready to move") + } + said := strings.Join(why, "\n") + for _, name := range []string{"anchor", "laptop"} { + if !strings.Contains(said, name) { + t.Errorf("the refusal does not name %s: %s", name, said) + } + } + + // Mint for one machine and it drops out of the complaint, which is how somebody works through it. + if _, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop", + }); err != nil { + t.Fatal(err) + } + state, err = readinessOf(ctx, inv) + if err != nil { + t.Fatal(err) + } + if !state.Credentialled["laptop"] { + t.Error("a machine that was minted a credential still reads as having none") + } +} diff --git a/go.mod b/go.mod index a827fad..47c7bce 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,23 @@ module github.com/novox/mesh-controller -go 1.25.0 +go 1.26.0 require ( github.com/jackc/pgx/v5 v5.10.0 github.com/rabbitmq/amqp091-go v1.14.0 - golang.org/x/crypto v0.55.0 + golang.org/x/crypto v0.57.0 ) require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - golang.org/x/net v0.57.0 // indirect - golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect ) diff --git a/go.sum b/go.sum index f1c77ed..e85a40e 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,14 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek= @@ -22,14 +30,24 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/broker/agreement.go b/internal/broker/agreement.go new file mode 100644 index 0000000..3f4fc6b --- /dev/null +++ b/internal/broker/agreement.go @@ -0,0 +1,125 @@ +package broker + +import ( + "fmt" + "sort" + "strings" +) + +// Do the emitters and the consumers of a catalogue agree? +// +// **The check that was missing** (novox/hq 04-ISSUES/127). Every manifest was individually +// well-formed and every derivation individually correct, and no cross-module subscription in the +// mesh matched anything: a consumer's declaration derived into a namespace nobody publishes to. +// Nothing failed, because a subscription that matches nothing is not an error — it is silence. +// +// The comparison has to be over the whole catalogue, because the two halves live in different +// manifests, and it cannot simply demand that every consumed event have a live emitter: a module +// may be installed long before the one whose events it wants. So the rule is narrower and still +// catches this: **where the emitter is present, it must emit what the consumer asked for.** + +// AConsumer is one module's interest in another's events, as this check needs it. +type AConsumer struct { + Module string + Consumes []string +} + +// AnEmitter is one module's events. +type AnEmitter struct { + Module string + Emits []string +} + +// Disagreements are the consumed events whose emitter is in the catalogue and does not emit them. +// +// Returned as sentences rather than as structs: every one of them is read by a person deciding +// whether a manifest or a catalogue is wrong, and a pair of names without the reason is a puzzle. +func Disagreements(emitters []AnEmitter, consumers []AConsumer, seats []DeclaredSeat) []string { + emits := map[string]map[string]bool{} + for _, e := range emitters { + if emits[e.Module] == nil { + emits[e.Module] = map[string]bool{} + } + for _, name := range e.Emits { + emits[e.Module][name] = true + } + } + // A seat's events are published by its holder under the seat's name, so a consumer naming the + // seat is naming something real even though no module declares it as its own. + for _, s := range seats { + if len(s.Emits) == 0 { + continue + } + if emits[s.Name] == nil { + emits[s.Name] = map[string]bool{} + } + for _, name := range s.Emits { + emits[s.Name][name] = true + } + } + + var out []string + for _, c := range consumers { + for _, pattern := range c.Consumes { + emitter, event, named := strings.Cut(pattern, ".") + // Every event from everyone, or every event from one module: both are deliberate and + // neither names a particular event to check. + if !named || emitter == "*" || emitter == catalogueTheRest || event == catalogueTheRest { + continue + } + known, present := emits[emitter] + if !present { + // Not installed here, which is ordinary: a module lives in its own repository and + // may be registered later. Nothing to compare, so nothing to say. + continue + } + if matchesAny(event, known) { + continue + } + out = append(out, fmt.Sprintf( + "%s consumes %q and %s emits %s — so that subscription would match nothing, and "+ + "nothing would report it", + c.Module, pattern, emitter, listOf(known))) + } + } + sort.Strings(out) + return out +} + +// matchesAny says whether one of an emitter's event names satisfies a consumer's pattern. +func matchesAny(pattern string, emitted map[string]bool) bool { + want := strings.Split(pattern, ".") + for name := range emitted { + if matches(want, strings.Split(name, ".")) { + return true + } + } + return false +} + +func matches(pattern, name []string) bool { + for i, part := range pattern { + if part == catalogueTheRest { + return i < len(name) + } + if i >= len(name) { + return false + } + if part != "*" && part != name[i] { + return false + } + } + return len(pattern) == len(name) +} + +func listOf(names map[string]bool) string { + if len(names) == 0 { + return "nothing" + } + out := make([]string, 0, len(names)) + for n := range names { + out = append(out, n) + } + sort.Strings(out) + return strings.Join(out, ", ") +} diff --git a/internal/broker/agreement_catalogue_test.go b/internal/broker/agreement_catalogue_test.go new file mode 100644 index 0000000..218904e --- /dev/null +++ b/internal/broker/agreement_catalogue_test.go @@ -0,0 +1,236 @@ +package broker + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Do the catalogue's emitters and consumers agree?** +// +// This is the check whose absence let issue 127 stand: every manifest was individually well-formed, +// every derivation individually correct, and no cross-module subscription in the mesh matched +// anything. A subscription that matches nothing is not an error — it is silence — so nothing +// anywhere reported it. +// +// It compares what one manifest asks to hear against what another says it emits. It cannot demand +// that every consumed event have a live emitter, because a module lives in its own repository and +// may be registered long before the one whose events it wants. Where the emitter *is* here, it must +// emit what the consumer asked for. +func TestTheCataloguesEmittersAndConsumersAgree(t *testing.T) { + emitters, consumers, seats := theCataloguesEvents(t) + + if bad := Disagreements(emitters, consumers, seats); len(bad) > 0 { + t.Fatalf("%d subscription(s) in the catalogue would match nothing:\n %s", + len(bad), strings.Join(bad, "\n ")) + } +} + +// And the check itself catches the thing it exists for, so it cannot pass by doing nothing. +func TestTheAgreementCheckCatchesASubscriptionThatMatchesNothing(t *testing.T) { + bad := Disagreements( + []AnEmitter{{Module: "builder", Emits: []string{"built"}}}, + []AConsumer{{Module: "mesh-catalog", Consumes: []string{"builder.finished"}}}, + nil) + if len(bad) != 1 { + t.Fatalf("a consumer asking for an event its emitter does not emit was not caught: %v", bad) + } + if !strings.Contains(bad[0], "builder.finished") || !strings.Contains(bad[0], "built") { + t.Fatalf("the report names neither what was asked for nor what is emitted: %s", bad[0]) + } + + // A module that is not here is not a disagreement: it may be registered later. + if bad := Disagreements(nil, + []AConsumer{{Module: "plex", Consumes: []string{"sonarr.download.completed"}}}, nil); len(bad) != 0 { + t.Fatalf("a consumer whose emitter is not installed was reported: %v", bad) + } + + // A wildcard over emitters is deliberate and names no particular event to check. + if bad := Disagreements([]AnEmitter{{Module: "sonarr", Emits: []string{"download.completed"}}}, + []AConsumer{{Module: "plex", Consumes: []string{"*.download.completed"}}}, nil); len(bad) != 0 { + t.Fatalf("a wildcard over emitters was reported: %v", bad) + } + + // A consumer of a role's event whose role does not emit it is caught, which is what stops the + // catalogue check above from passing by knowing nothing about roles. + if bad := Disagreements(nil, + []AConsumer{{Module: "mesh-catalog", Consumes: []string{"mesh-build-machine.finished"}}}, + []DeclaredSeat{{Name: "mesh-build-machine", Emits: []string{"built"}}}); len(bad) != 1 { + t.Fatalf("a consumer of a role event the role does not emit was not caught: %v", bad) + } + + // An event published under a seat's name is real even though no module declares it as its own. + if bad := Disagreements(nil, + []AConsumer{{Module: "watcher", Consumes: []string{"mesh-artifact-store.image.pushed"}}}, + []DeclaredSeat{{Name: "the-artifact-store", Emits: []string{"image.pushed"}}}); len(bad) != 0 { + t.Fatalf("an event a seat emits was reported as matching nothing: %v", bad) + } +} + +func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat) { + t.Helper() + root := filepath.Join("..", "..", "..", "mesh-catalog", "modules") + entries, err := os.ReadDir(root) + if err != nil { + t.Skipf("catalogue sibling not present: %v", err) + } + var emitters []AnEmitter + var consumers []AConsumer + // The mesh's own roles, which emit under the seat's name rather than any module's (novox/hq + // ADR 0121). Without these the check skips every consumer of a role's event as "the emitter is + // not installed" — which is how it passed vacuously the first time one existed. + var seats []DeclaredSeat + for _, own := range catalogue.SeatsWithAProtocol() { + seats = append(seats, DeclaredSeat{Name: own.Name, Accepts: own.Accepts, Emits: own.Emits}) + } + for _, e := range entries { + if !e.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json")) + if err != nil { + continue + } + var m struct { + Module string `json:"module"` + Emits []string `json:"emits"` + Consumes []string `json:"consumes"` + Seats []struct { + Name string `json:"name"` + Emits []string `json:"emits"` + } `json:"seats"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", e.Name(), err) + } + if len(m.Emits) > 0 { + emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits}) + } + if len(m.Consumes) > 0 { + consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes}) + } + for _, s := range m.Seats { + seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits}) + } + } + if len(emitters) == 0 { + t.Skip("no manifests found beside this checkout") + } + return emitters, consumers, seats +} + +// **Do the derived subjects meet, not just the names?** +// +// The check above compares what a consumer asks for against what an emitter says it emits, by name. It +// passed while the catalogue's subscription pointed at `mesh.mod.mesh-build-machine.event.built` — a +// module namespace for a role's event, which no emitter owns. The names agreed; the subjects did not, +// and the graph stayed empty. +// +// So this compares the thing that actually has to match: the subject a consumer subscribes against the +// subject an emitter publishes. It is the last place the two halves can be held together, because +// after this the server is the only thing that knows and it says nothing — a subscription that matches +// nothing is silence. +func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) { + emitters, consumers, seats := theCataloguesEvents(t) + + // Every subject something publishes: a module's own events, and the events of every role. + published := map[string]bool{} + for _, e := range emitters { + for _, name := range e.Emits { + published["mesh.mod."+e.Module+".event."+name] = true + } + } + for _, s := range seats { + for _, name := range s.Emits { + published["mesh.seat."+s.Name+".event."+name] = true + } + } + + byName := map[string]DeclaredSeat{} + for _, s := range seats { + byName[s.Name] = s + } + + var lonely []string + for _, c := range consumers { + principal := Principal{Kind: KindModule, Node: "one", Module: c.Module, PasswordHash: "x"} + for _, want := range c.Consumes { + emitter, event, named := strings.Cut(want, ".") + if named { + if s, isASeat := byName[emitter]; isASeat { + principal.Watches = append(principal.Watches, + Seat{Name: s.Name, Emits: []string{event}}) + continue + } + } + principal.Consumes = append(principal.Consumes, want) + } + perms, err := PermissionsFor(principal) + if err != nil { + t.Fatalf("%s: %v", c.Module, err) + } + for _, subject := range perms.Subscribe { + if !strings.Contains(subject, ".event.") { + continue + } + if reaches(subject, published) { + continue + } + // A wildcard over emitters reaches whatever arrives later, and an emitter that is not + // installed is ordinary — both are already excused by the check above, so only a subject + // that can never match anything gets here. + if strings.Contains(subject, "*") || strings.Contains(subject, ">") { + continue + } + lonely = append(lonely, c.Module+" subscribes "+subject+", which nothing publishes") + } + } + if len(lonely) > 0 { + sort.Strings(lonely) + t.Fatalf("%d subscription(s) derive to a subject no emitter owns:\n %s", + len(lonely), strings.Join(lonely, "\n ")) + } +} + +// And it catches the thing it exists for: a role's event read as a module's. +func TestTheDerivedSubjectCheckCatchesARolesEventReadAsAModules(t *testing.T) { + published := map[string]bool{"mesh.seat.mesh-build-machine.event.built": true} + // What the derivation produced before a consumed seat name was resolved as one. + if reaches("mesh.mod.mesh-build-machine.event.built", published) { + t.Fatal("a module namespace was treated as reaching a role's event, which is the bug") + } + // And the corrected one does reach it. + if !reaches("mesh.seat.mesh-build-machine.event.built", published) { + t.Fatal("the role's own subject does not reach the role's event") + } +} + +// reaches says whether a subscribed subject admits any published one. +func reaches(subject string, published map[string]bool) bool { + for p := range published { + if admitsSubject(strings.Split(subject, "."), strings.Split(p, ".")) { + return true + } + } + return false +} + +func admitsSubject(pattern, subject []string) bool { + for i, token := range pattern { + if token == ">" { + return i < len(subject) + } + if i >= len(subject) { + return false + } + if token != "*" && token != subject[i] { + return false + } + } + return len(pattern) == len(subject) +} diff --git a/internal/broker/derived.go b/internal/broker/derived.go new file mode 100644 index 0000000..f5f514a --- /dev/null +++ b/internal/broker/derived.go @@ -0,0 +1,237 @@ +package broker + +import ( + "fmt" + "sort" + "strings" +) + +// Streams and consumers derived from what modules declare. +// +// The mesh's own four exist before any module does (streams.go). Everything here is the other +// half: a seat's stream comes into being when the module declaring it is **registered**, and a +// consumer when a module is **assigned** — which is why ADR 0116's task 1.4 had to be narrowed to +// the foundation set. Neither has happened at genesis. +// +// All of it is a pure function of declarations. The controller is still the only writer; this is +// only what it writes. + +// A Consumer is a durable subscription the controller creates on a module's behalf. A module +// declares what it reacts to, never how delivery works, so it does not name these and cannot +// misconfigure them. +type Consumer struct { + Name string + Stream string + // Filters are the subjects this consumer receives. One consumer per module with several + // filters, rather than one per consumed event: its ack subject is derived from its name, and + // a module with five consumers would need five ack permissions to ack its own deliveries. + Filters []string + // Queue is the queue group, set for a seat's worker so that "exactly one holder" survives a + // seat later being relaxed to several. Authority and delivery are kept separate on purpose. + Queue string + // Push asks the server to deliver to a subject rather than wait to be pulled. + // + // For the mesh's own consumer, where the controller wants every message to arrive in the one + // loop it already runs: pulling would mean a second goroutine fetching batches and handing + // them over, and a loop that acts on one message at a time is the property the store window + // depends on. A queue group implies this, because a group has nothing to pull from. + Push bool + // AckWaitSeconds before an unacknowledged delivery is redelivered. + AckWaitSeconds int + // MaxDeliver before the message is dead-lettered; zero for the mesh's default. + MaxDeliver int + Why string +} + +// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than +// the module holding it, because the holder can change and the queued work must not care — which +// is the whole reason a caller addresses a seat instead of a module. +func seatStreamName(seat string) string { return "SEAT_" + upperSnake(seat) } + +// SeatStreams is one work queue per declared seat, created when the declaring module is +// registered rather than when it is assigned. +// +// **The stream exists before anyone holds the seat, and that is the point.** Work queues until a +// holder appears, so installing the telegram module a week after something started sending to it +// flushes the backlog instead of having lost it. A stream created at assignment would make "the +// holder is not here yet" mean "your messages are gone". +func SeatStreams(seats []DeclaredSeat) []Stream { + sorted := append([]DeclaredSeat(nil), seats...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) + + var out []Stream + for _, s := range sorted { + if len(s.Accepts) == 0 { + // A seat that only emits and serves needs no stream: its events ride EVENTS and its + // tools are core request/reply, which is never persisted. + continue + } + retain := s.RetainSeconds + if retain == 0 { + retain = 7 * 24 * 60 * 60 + } + out = append(out, Stream{ + Name: seatStreamName(s.Name), + Subjects: []string{"mesh.seat." + s.Name + ".accept.>"}, + Retention: RetentionWorkQueue, + MaxAge: retain, + Why: fmt.Sprintf("work submitted to the %s seat; one holder consumes it, and it "+ + "queues while nobody does", s.Name), + }) + } + return out +} + +// A DeclaredSeat is a seat as the catalogue knows it. Mirrored here rather than imported so this +// package stays free of the catalogue's own types — the same reason the host mirrors the +// contracts instead of importing the sdk. +type DeclaredSeat struct { + Name string + Accepts []string + // Emits are the verbs the seat's holder publishes under the seat's own name. An event about a + // role belongs here rather than in the holder's namespace, because the name then outlives + // whoever fills it (novox/hq ADR 0121, 04-ISSUES/127). + Emits []string + // Serves are the verbs the holder answers, request and reply. + Serves []string + RetainSeconds int +} + +// ConsumerFor is the durable consumer a module's declarations imply, or false when it subscribes +// to nothing and needs none. +// +// One per module, with every consumed subject as a filter, because its ack permission is derived +// from its name: a module with a consumer per event would need an ack permission per consumer, +// and the permission list would stop being derivable from the declaration. +func ConsumerFor(p Principal) (Consumer, bool) { + // A module that reacts to anything — a module's events or a role's (novox/hq ADR 0121). Watching + // a role was missing here, so the one module that does it got no consumer at all: it started, + // connected, and its graph stayed empty with nothing anywhere reporting why. + if p.Kind != KindModule || (len(p.Consumes) == 0 && len(p.Watches) == 0) { + return Consumer{}, false + } + perms, err := PermissionsFor(p) + if err != nil { + return Consumer{}, false + } + // Events, wherever they live: a module's own namespace, and the namespace of any role it watches + // (novox/hq ADR 0121). Tool subjects and inboxes are subscribed directly and are not a consumer's + // business, which is why this is a filter and not the whole list. + var filters []string + for _, s := range perms.Subscribe { + if strings.Contains(s, ".event.") { + filters = append(filters, s) + } + } + if len(filters) == 0 { + return Consumer{}, false + } + sort.Strings(filters) + return Consumer{ + Name: consumerDurable(p), + Stream: consumerStream(p), + Filters: filters, + AckWaitSeconds: 30, + MaxDeliver: 5, + Why: "what " + p.Module + " declared it consumes; after max-deliver it dead-letters", + }, true +} + +// HolderConsumerFor is the worker a seat's holder gets on that seat's work queue. +// +// **A queue group even though the seat guarantees one holder.** The seat is *authority* — who may +// be the telegram sender — and the queue group is *delivery*. Tie delivery to the seat and the +// day somebody allows two holders for throughput, every message is processed twice with nothing +// reporting it. Kept separate, relaxing one changes nothing about the other. +func HolderConsumerFor(node, module string, seat DeclaredSeat) (Consumer, bool) { + if len(seat.Accepts) == 0 { + return Consumer{}, false + } + return Consumer{ + Name: "SEAT_" + upperSnake(seat.Name) + "_worker", + Stream: seatStreamName(seat.Name), + Filters: []string{"mesh.seat." + seat.Name + ".accept.>"}, + Queue: "holders", + AckWaitSeconds: 60, + MaxDeliver: 5, + Why: fmt.Sprintf("%s on %s holds %s; it acknowledges after the work is done, so a "+ + "crash mid-work redelivers rather than loses", module, node, seat.Name), + }, true +} + +// NodeConsumer is the durable consumer a node reads its own declaration through. +// +// **Derived from a node existing, and created by the controller, because a host cannot create it.** +// A host's account may subscribe its own declaration subject and publish its own ack subject, and +// reaches no part of the JetStream API — which is correct (the controller is the only writer of +// consumer definitions, design 25 §3) and means the consumer must be waiting before the host binds +// to it. Named after the node, because the node's ack grant is `$JS.ACK.NODES..>` and a +// consumer named anything else is one the host cannot acknowledge a delivery from. +// +// **No max-deliver, and a long ack wait.** A declaration is settled only after the node has applied +// it and reported, which is minutes on a machine pulling images; and a declaration the mesh cannot +// get a node to accept is not one to dead-letter, because the stream keeps only the newest per node +// anyway — so there is exactly one message per node to redeliver, for as long as that node is away. +func NodeConsumer(node string) Consumer { + return Consumer{ + Name: node, + Stream: "NODES", + Filters: []string{"mesh.node." + node + ".declare"}, + Push: true, + AckWaitSeconds: 300, + Why: "how " + node + " hears what it should be; last-per-subject, so a node that was away " + + "gets exactly the current declaration and nothing older", + } +} + +// AssertNodeConsumers brings every known node's declaration consumer into being. +// +// Asserted on start as well as created at enrolment, for the reason the streams are: a mesh raised +// from a restored backup, or one whose bus was recreated, has node records and no consumers, and a +// node whose consumer is missing hears nothing while everything else about it looks correct. +func AssertNodeConsumers(e Ensurer, nodes []string) error { + for _, n := range nodes { + if err := e.EnsureConsumer(NodeConsumer(n)); err != nil { + return fmt.Errorf("asserting how %s hears its declaration: %w", n, err) + } + } + return nil +} + +// AllOverlaps reports subject filters claimed by more than one stream, across the mesh's own and +// every derived one. +// +// NATS refuses an overlapping stream rather than merging it (verified against nats-server 2.10: +// "subjects overlap with an existing stream"), so this is not a subtle divergence — it is a +// registration that fails. Catching it here names both streams, before a half-applied mesh does. +func AllOverlaps(seats []DeclaredSeat) []string { + all := append(MeshStreams(), SeatStreams(seats)...) + seen := map[string]string{} + var clashes []string + for _, s := range all { + for _, subject := range s.Subjects { + if first, ok := seen[subject]; ok { + clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject)) + continue + } + seen[subject] = s.Name + } + } + sort.Strings(clashes) + return clashes +} + +// upperSnake makes a stream name from a seat name. NATS stream names may not contain a dot, +// a space or a wildcard, and a hyphen is legal but reads badly beside the mesh's own. +func upperSnake(s string) string { + out := []rune(s) + for i, r := range out { + switch { + case r >= 'a' && r <= 'z': + out[i] = r - 32 + case r == '-' || r == '.': + out[i] = '_' + } + } + return string(out) +} diff --git a/internal/broker/derived_test.go b/internal/broker/derived_test.go new file mode 100644 index 0000000..b4529d4 --- /dev/null +++ b/internal/broker/derived_test.go @@ -0,0 +1,155 @@ +package broker + +import ( + "strings" + "testing" +) + +func telegramSeat() DeclaredSeat { + return DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}} +} + +// The stream exists from registration, not assignment: work queues until a holder appears, so +// installing the module a week later flushes the backlog rather than having lost it. +func TestASeatGetsAWorkQueueOfItsOwn(t *testing.T) { + got := SeatStreams([]DeclaredSeat{telegramSeat()}) + if len(got) != 1 { + t.Fatalf("expected one stream, got %d", len(got)) + } + s := got[0] + if s.Retention != RetentionWorkQueue { + t.Fatalf("a seat's inbound queue retains as %q; one holder must take each message once", s.Retention) + } + if s.Subjects[0] != "mesh.seat.telegram-sender.accept.>" { + t.Fatalf("filters on %v", s.Subjects) + } +} + +// A seat that only emits and serves needs no stream: its events ride EVENTS and its tools are +// core request/reply, which is never persisted. +func TestASeatThatAcceptsNothingGetsNoStream(t *testing.T) { + if got := SeatStreams([]DeclaredSeat{{Name: "announcer"}}); len(got) != 0 { + t.Fatalf("a seat with no inbound work got %d stream(s)", len(got)) + } +} + +// Retention belongs to whoever owns the namespace, and a seat owns its own. +func TestASeatsRetentionIsItsOwn(t *testing.T) { + s := SeatStreams([]DeclaredSeat{{Name: "slow", Accepts: []string{"work"}, RetainSeconds: 30 * 24 * 60 * 60}}) + if s[0].MaxAge != 30*24*60*60 { + t.Fatalf("the seat's declared retention was not used: %d", s[0].MaxAge) + } + d := SeatStreams([]DeclaredSeat{telegramSeat()}) + if d[0].MaxAge == 0 { + t.Fatal("a seat that declares no retention got an unbounded queue") + } +} + +// NATS refuses an overlapping stream outright, so a clash here is a registration that fails. +func TestNoDerivedStreamOverlapsTheMeshsOwn(t *testing.T) { + seats := []DeclaredSeat{telegramSeat(), {Name: "licensing-master", Accepts: []string{"report"}}} + if c := AllOverlaps(seats); len(c) != 0 { + t.Fatalf("overlapping filters: %v", c) + } +} + +// One consumer per module, with every consumed subject as a filter — because its ack permission +// is derived from its name, and a consumer per event would need an ack permission per consumer. +func TestAModuleGetsOneConsumerCarryingEveryFilter(t *testing.T) { + c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"}) + if !ok { + t.Fatal("a module that consumes got no consumer") + } + if len(c.Filters) != 2 { + t.Fatalf("expected both subjects as filters, got %v", c.Filters) + } + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed"}, PasswordHash: "x"}) + ack := "$JS.ACK." + c.Stream + "." + c.Name + ".>" + found := false + for _, p := range perms.Publish { + if p == ack { + found = true + } + } + if !found { + t.Fatalf("the consumer is named %q but the ack permission is %v; a module could not ack "+ + "its own deliveries", c.Name, perms.Publish) + } +} + +// A module that subscribes to nothing needs no consumer, and creating one would leave an object +// nothing reads and everything has to maintain. +func TestAModuleThatConsumesNothingGetsNoConsumer(t *testing.T) { + if _, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "shop", + Emits: []string{"order.placed"}, PasswordHash: "x"}); ok { + t.Fatal("a pure emitter got a consumer") + } +} + +// The seat is authority and the queue group is delivery. Tie them together and the day somebody +// allows two holders, every message is processed twice with nothing reporting it. +func TestAHoldersWorkerUsesAQueueGroupAnyway(t *testing.T) { + c, ok := HolderConsumerFor("one", "telegram", telegramSeat()) + if !ok { + t.Fatal("the holder of a seat with inbound work got no worker") + } + if c.Queue == "" { + t.Fatal("the worker is not in a queue group, so a second holder would double-process") + } + if c.Stream != "SEAT_TELEGRAM_SENDER" { + t.Fatalf("the worker reads %q, not the seat's own stream", c.Stream) + } + if c.MaxDeliver == 0 { + t.Fatal("a failing worker would redeliver forever rather than dead-letter") + } +} + +// The stream is named after the seat, not its holder: the holder can change and the queued work +// must not care. +func TestASeatsStreamIsNamedAfterTheSeat(t *testing.T) { + name := seatStreamName("telegram-sender") + if strings.Contains(name, "telegram-sender") { + t.Fatalf("%q keeps characters a stream name may not hold", name) + } + if name != "SEAT_TELEGRAM_SENDER" { + t.Fatalf("unexpected stream name %q", name) + } +} + +// A node hears its declaration through a consumer only the controller can make. +// +// The three things that would each break it silently: a name other than the node's is one the host +// cannot acknowledge a delivery from, because its ack grant is derived from the node's name; a +// filter other than its own declaration subject is a node reading another's; and a pull consumer is +// one the host cannot bind a channel to without creating something, which it has no authority for. +func TestANodesDeclarationConsumerIsWhatItsOwnGrantAllows(t *testing.T) { + c := NodeConsumer("anchor") + if c.Name != "anchor" { + t.Fatalf("named %q, so the node cannot ack from it: its grant is $JS.ACK.NODES.anchor.>", c.Name) + } + if c.Stream != "NODES" { + t.Fatalf("on stream %q rather than the one declarations live in", c.Stream) + } + if len(c.Filters) != 1 || c.Filters[0] != "mesh.node.anchor.declare" { + t.Fatalf("filters %v, which is not this node's own declaration and nothing else", c.Filters) + } + if !c.Push { + t.Fatal("pulled, which a host cannot do: pulling needs the JetStream API and a host reaches none of it") + } + if c.MaxDeliver != 0 { + t.Fatalf("max-deliver %d: a declaration a node has not taken yet is not one to dead-letter, "+ + "because the stream holds exactly one per node", c.MaxDeliver) + } + + // And the grant the node actually gets has to match, or none of the above matters. + perms, err := PermissionsFor(Principal{Kind: KindNode, Node: "anchor"}) + if err != nil { + t.Fatal(err) + } + // Without the ack grant every declaration a node receives is redelivered for ever; without the + // subscribe grant its consumer delivers to nobody. + has(t, perms.Publish, "$JS.ACK.NODES."+c.Name+".>") + has(t, perms.Subscribe, c.Filters[0]) +} diff --git a/internal/broker/genesis_template_test.go b/internal/broker/genesis_template_test.go new file mode 100644 index 0000000..1260f85 --- /dev/null +++ b/internal/broker/genesis_template_test.go @@ -0,0 +1,126 @@ +package broker + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "golang.org/x/crypto/bcrypt" +) + +// **The first user list the installer carries must be the one the controller would compose.** +// +// At genesis there is no mesh to write the bus's user list, so the installer carries one: the +// controller's own account, at a bootstrap password, the way the store is reached at +// `postgres:bootstrap` (novox/hq design 25 §4, task 1.7). It is written by hand in a template and +// derived in code here, which is two statements of one fact — so this compares them. +// +// Getting it wrong is the worst kind of silent: a controller whose carried permissions are narrower +// than the ones it derives comes up, connects, and is refused on the first thing it tries, with an +// authorisation error that names a subject and not the template that forgot it. And a mesh cannot be +// raised twice to find out. +func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T) { + accounts := theCarriedAccounts(t) + + want, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + carriedPub := subjectsIn(accounts, "publish") + carriedSub := subjectsIn(accounts, "subscribe") + + if diff := missing(want.Publish, carriedPub); len(diff) > 0 { + t.Errorf("the installer's user list does not let the controller publish %v — it would come up "+ + "and be refused on the first thing it tried", diff) + } + if diff := missing(want.Subscribe, carriedSub); len(diff) > 0 { + t.Errorf("the installer's user list does not let the controller subscribe %v", diff) + } + // And nothing wider than what it derives, or genesis quietly grants a privilege the composition + // takes away again on the first push. + if diff := missing(carriedPub, want.Publish); len(diff) > 0 { + t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff) + } + if diff := missing(carriedSub, want.Subscribe); len(diff) > 0 { + t.Errorf("the installer's user list lets the controller subscribe %v, which it does not derive", diff) + } + + // The credential is the bootstrap one and the hash really is of it, because a hash of something + // else is a controller that cannot log in to the bus it was just given. + hash := regexp.MustCompile(`\$2[aby]?\$[0-9]+\$[A-Za-z0-9./]{53}`).FindString(accounts) + if hash == "" { + t.Fatal("the installer's user list carries no password hash") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { + t.Fatalf("the carried hash does not verify the bootstrap credential the template also carries: %v", err) + } +} + +// theCarriedAccounts is the accounts file the installer's template writes at genesis. +func theCarriedAccounts(t *testing.T) string { + t.Helper() + path := filepath.Join("..", "..", "..", "mesh-host", "examples", "foundation-first-node-nats.lock") + raw, err := os.ReadFile(path) + if err != nil { + t.Skipf("the host's checkout is not beside this one: %v", err) + } + // The template is JSON with line comments, which is how every one of them is written. + var lines []string + for _, l := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(l), "//") { + lines = append(lines, l) + } + } + var bundle struct { + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &bundle); err != nil { + t.Fatalf("the template is not readable: %v", err) + } + for _, r := range bundle.Resources { + if r["id"] == "bus-accounts" { + content, _ := r["content"].(string) + if content == "" { + t.Fatal("the template's accounts file is empty, so the bus would refuse every connection") + } + return content + } + } + t.Fatal("the template carries no accounts file, so a mesh raised from it has a bus nobody may use") + return "" +} + +// subjectsIn reads one allow-list out of a composed accounts file. +func subjectsIn(accounts, which string) []string { + found := regexp.MustCompile(which + `: \{ allow: \[([^\]]*)\]`).FindStringSubmatch(accounts) + if len(found) != 2 { + return nil + } + var out []string + for _, part := range strings.Split(found[1], ",") { + if s := strings.Trim(strings.TrimSpace(part), `"`); s != "" { + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +// missing is what is in want and not in got. +func missing(want, got []string) []string { + have := map[string]bool{} + for _, g := range got { + have[g] = true + } + var out []string + for _, w := range want { + if !have[w] { + out = append(out, w) + } + } + return out +} diff --git a/internal/broker/jetstream.go b/internal/broker/jetstream.go new file mode 100644 index 0000000..1867e21 --- /dev/null +++ b/internal/broker/jetstream.go @@ -0,0 +1,150 @@ +package broker + +import ( + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" +) + +// The JetStream side of the controller: the one place the mesh's streams and consumers are +// actually created. +// +// Everything that decides *what* they are is pure and lives beside this (streams.go, derived.go). +// This is only the part that talks to a server, kept small on purpose: a bug in a subject filter +// should be findable in a unit test, and only a bug in "did the server accept it" should need one +// running. + +// A JetStream is a connection to the bus, as the controller uses it. +type JetStream struct { + conn *nats.Conn + js nats.JetStreamContext +} + +// Dial connects and returns the controller's JetStream handle. +func Dial(url string, opts ...nats.Option) (*JetStream, error) { + // A name, because a connection nobody can identify in the server's own monitoring is one + // nobody can attribute a problem to. + opts = append(opts, nats.Name("mesh-controller"), nats.Timeout(10*time.Second)) + conn, err := nats.Connect(url, opts...) + if err != nil { + return nil, fmt.Errorf("connecting to the bus at %s: %w", url, err) + } + js, err := conn.JetStream() + if err != nil { + conn.Close() + return nil, fmt.Errorf("the bus at %s has no JetStream: %w", url, err) + } + return &JetStream{conn: conn, js: js}, nil +} + +// Conn is the connection itself, for what the mesh keeps off JetStream on purpose — a heartbeat, +// a tool call — where a lost message is answered by the next one or by a timeout the caller +// already handles (design 25 §3). +func (j *JetStream) Conn() *nats.Conn { return j.conn } + +// Context is the JetStream handle, for subscribing to what the consumers above define. +func (j *JetStream) Context() nats.JetStreamContext { return j.js } + +func (j *JetStream) Close() { + if j.conn != nil { + j.conn.Close() + } +} + +// EnsureStream creates the stream if it is absent and brings it to match if it is present. +// +// **Idempotent, because the controller asserts on every start** rather than creating once at +// genesis: a stream somebody deleted, or a mesh raised from a restored backup, has to converge +// rather than run without the guarantee its messages assume. +// +// An update, not a delete and recreate. Recreating would discard every message the stream holds +// and every consumer's position in it — which for CONTROL means the pushes being held through a +// store restart, exactly the guarantee the stream exists for. +func (j *JetStream) EnsureStream(s Stream) error { + want := &nats.StreamConfig{ + Name: s.Name, + Subjects: s.Subjects, + Retention: retentionOf(s.Retention), + MaxAge: time.Duration(s.MaxAge) * time.Second, + MaxMsgsPerSubject: int64(s.MaxMsgsPerSubject), + Description: s.Why, + } + if s.Retention == RetentionLastPerSubject { + // Last-per-subject is a limits stream with one message kept per subject, not a + // retention policy of its own — the state shape, spelled the way the server spells it. + want.Retention = nats.LimitsPolicy + want.MaxMsgsPerSubject = 1 + want.MaxAge = 0 + } + + switch _, err := j.js.StreamInfo(s.Name); { + case err == nil: + if _, err := j.js.UpdateStream(want); err != nil { + return fmt.Errorf("bringing stream %s to match: %w", s.Name, err) + } + return nil + case errors.Is(err, nats.ErrStreamNotFound): + if _, err := j.js.AddStream(want); err != nil { + return fmt.Errorf("creating stream %s: %w", s.Name, err) + } + return nil + default: + return fmt.Errorf("asking about stream %s: %w", s.Name, err) + } +} + +// EnsureConsumer creates or updates one durable consumer. +// +// Explicit acknowledgement throughout: a consumer that acknowledges on delivery cannot redeliver +// work its holder died in the middle of, which is the whole difference between a queue and a +// firehose. +func (j *JetStream) EnsureConsumer(c Consumer) error { + want := &nats.ConsumerConfig{ + Durable: c.Name, + AckPolicy: nats.AckExplicitPolicy, + AckWait: time.Duration(c.AckWaitSeconds) * time.Second, + MaxDeliver: c.MaxDeliver, + DeliverGroup: c.Queue, + DeliverSubject: "", + Description: c.Why, + } + switch len(c.Filters) { + case 0: + case 1: + want.FilterSubject = c.Filters[0] + default: + want.FilterSubjects = c.Filters + } + // A queue group needs a delivery subject: a pull consumer has no group, and declaring one + // without the other is refused by the server with a message that does not say which half is + // missing. + if c.Queue != "" || c.Push { + want.DeliverSubject = "_DELIVER." + c.Name + } + + switch _, err := j.js.ConsumerInfo(c.Stream, c.Name); { + case err == nil: + if _, err := j.js.UpdateConsumer(c.Stream, want); err != nil { + return fmt.Errorf("bringing consumer %s on %s to match: %w", c.Name, c.Stream, err) + } + return nil + case errors.Is(err, nats.ErrConsumerNotFound): + if _, err := j.js.AddConsumer(c.Stream, want); err != nil { + return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err) + } + return nil + default: + return fmt.Errorf("asking about consumer %s on %s: %w", c.Name, c.Stream, err) + } +} + +func retentionOf(r Retention) nats.RetentionPolicy { + switch r { + case RetentionWorkQueue: + return nats.WorkQueuePolicy + default: + return nats.LimitsPolicy + } +} diff --git a/internal/broker/jetstream_test.go b/internal/broker/jetstream_test.go new file mode 100644 index 0000000..396cf16 --- /dev/null +++ b/internal/broker/jetstream_test.go @@ -0,0 +1,71 @@ +package broker + +import ( + "os" + "testing" +) + +// Against a real server, because the questions here are all "does the server accept this" — +// which a mock would answer by agreeing with whatever this file already believes. +// +// Skipped unless MESH_TEST_NATS names one, so the ordinary suite stays fast and offline: +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/broker/ -run TestAgainstARealServer +func TestAgainstARealServer(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := Dial(url) + if err != nil { + t.Fatal(err) + } + defer js.Close() + + t.Run("the mesh's own streams are accepted", func(t *testing.T) { + if err := AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + }) + + t.Run("asserting again changes nothing and fails nothing", func(t *testing.T) { + if err := AssertMeshStreams(js); err != nil { + t.Fatalf("the second assertion failed, so the controller cannot restart: %v", err) + } + }) + + t.Run("a seat's work queue is accepted beside them", func(t *testing.T) { + seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}} + for _, s := range SeatStreams(seats) { + if err := js.EnsureStream(s); err != nil { + t.Fatal(err) + } + } + if c := AllOverlaps(seats); len(c) != 0 { + t.Fatalf("overlaps the server would refuse: %v", c) + } + }) + + t.Run("a module's consumer is accepted and is idempotent", func(t *testing.T) { + c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed", "billing.invoice.sent"}, PasswordHash: "x"}) + if !ok { + t.Fatal("no consumer derived") + } + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + if err := js.EnsureConsumer(c); err != nil { + t.Fatalf("the second assertion failed: %v", err) + } + }) + + t.Run("a holder's worker is accepted with its queue group", func(t *testing.T) { + c, _ := HolderConsumerFor("one", "telegram", + DeclaredSeat{Name: "telegram-sender", Accepts: []string{"send"}}) + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + }) +} diff --git a/internal/broker/nats.go b/internal/broker/nats.go new file mode 100644 index 0000000..3c18343 --- /dev/null +++ b/internal/broker/nats.go @@ -0,0 +1,547 @@ +// Composing the bus's own configuration. +// +// An account is *composed*, never called for: the controller writes accounts, users and +// per-subject permissions into one file the host keeps current, and the server reloads it in +// place (novox/hq ADR 0106 — never through a management API; design 25 §4). +// +// Everything here is pure. Given the principals, it returns the file's text — so the whole of the +// mesh's authority model is testable as strings, with no server. +// +// **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042 +// reserves a module's origin — it publishes only under its own name — and here that is a refusal +// rather than something a library promises. +package broker + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strings" +) + +// A Kind is what a principal is, which decides the shape of its authority rather than its +// contents: a module's comes from its declaration, a host's from its node, and the controller's +// and the enrolment user's are fixed. +type Kind string + +const ( + KindModule Kind = "module" + KindNode Kind = "node" + KindController Kind = "controller" + KindEnrolment Kind = "enrolment" + // KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its + // authority is a list of tools and nothing else — not control, not declarations, not builds, + // and no ability to answer anything, because a person asks. + KindPerson Kind = "person" +) + +// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it +// emits (novox/hq ADR 0118, design 29 §5). +type Seat struct { + Name string + Accepts []string + Emits []string + Serves []string + Versions []string // protocol versions served beside the current one; empty for v1 only +} + +// A Principal is one user of the bus. Its permissions are derived from what it declares and +// nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats +// it holds, and the seats it uses. +type Principal struct { + Kind Kind + Node string + Module string + + Emits []string + Consumes []string + Serves []string + + Holds []Seat + Uses []Seat + // Watches are seats whose events this principal consumes. Separate from Consumes because a + // role's event lives under the seat's namespace and not a module's, and this package cannot tell + // a seat's name from a module's by looking at it — whoever resolved the declaration can, and + // does (novox/hq ADR 0121). + // + // **Found by a consumer reading nothing.** The catalogue consumes the build machine's outcome; + // with that name read as a module's, its subscription pointed at `mesh.mod.mesh-build-machine.…`, + // a namespace no such module owns. Every service started and the graph stayed empty. + Watches []Seat + + // Invokes are the tools a person may call, as `.`; a single `*` is every tool, + // for an administrator. Only meaningful for KindPerson. + // + // **A list, not a role.** A person is not a module and holds no seat: nothing is addressed + // to them, nothing is delivered to them, and they have no durable consumer to acknowledge. + // What they have is permission to ask. + Invokes []string + + // PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal + // and never appears here: this file is written to a node's disk and read by a server, and a + // secret that can be read from a configuration file is a secret with a wider blast radius + // than the one it protects (novox/hq design 29 §10). + PasswordHash string +} + +// meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than +// derived from the seat set: the controller is not a module and declares no `uses`, so its side of a +// seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable. +var meshSeatsTheControllerUses = []string{"mesh-build-machine"} + +// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the +// controller may answer an enrolment is derived from the same constant the user is named from. +const enrolmentPrefix = "enrol" + +// safeSubject refuses anything that would change the meaning of a subject rather than sit inside +// one. A name carrying a dot would silently widen a permission by adding a token; a name carrying +// `>` or `*` would widen it to a wildcard, which is the whole authority model gone. +var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// Username is how a principal is named to the server. The node is part of it, so the same module +// on two machines holds two users, each sealed to its own — the rule management.go already +// applies, kept. +func (p Principal) Username() string { + switch p.Kind { + case KindPerson: + return "person." + p.Module + case KindModule: + return p.Node + "." + p.Module + case KindNode: + return "node." + p.Node + case KindController: + return "controller" + case KindEnrolment: + // Per token, not one shared user. **The inbox is the reason**: with a single `enrolment` + // user every machine enrolling at once could read every other's answer, and an answer + // carries that node's credentials sealed to it. Design 25 §6 says the inbox a token + // derives, and a permission belongs to a user, so the user is per token. + // + // Named after the node, which **is** the token's id: a token is issued for a node record, + // the mesh holds one live claim per record, and the node's name is the one identifier both + // sides already have before anything else is agreed. It is also exactly what the other + // transport does, where the account is named after the node and the secret is its password. + return enrolmentPrefix + "." + p.Node + } + return "" +} + +// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25 +// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's +// inbox is derived from its own identity and its permissions name that prefix and no other. +func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" } + +// Permissions is what a principal may publish and subscribe, and whether it may answer. +type Permissions struct { + Publish []string + Subscribe []string + // AllowResponses lets a principal reply to a request it received, on the reply subject that + // request carried, once. + // + // **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If + // every user's inbox is private to it, a module serving a tool cannot publish the answer — + // the answer goes to the *caller's* inbox, which the responder has no permission for. The two + // ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4 + // refuses, or this: the server itself permits one reply to the subject of a message the user + // actually received, and nothing else. The authority is bounded by having been asked. + AllowResponses bool +} + +// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided: +// a permission that cannot be derived from a declaration is a permission nobody can explain. +func PermissionsFor(p Principal) (Permissions, error) { + for _, part := range []struct{ what, value string }{ + {"node", p.Node}, {"module", p.Module}, + } { + if part.value == "" { + continue + } + if !safeSubject.MatchString(part.value) { + return Permissions{}, fmt.Errorf( + "%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value) + } + } + + var pub, sub []string + switch p.Kind { + case KindController: + // The controller owns the mesh's own traffic and the streams. It is the only writer of + // stream definitions (design 25 §3), so it alone reaches the JetStream API. + pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"} + sub = []string{"mesh.control.>", "$JS.API.>"} + + // Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A + // build is the one today: the controller asks, and reads the answer from the seat's event + // like the catalogue does — which is why no holder needs to publish into anybody's inbox. + for _, seat := range meshSeatsTheControllerUses { + pub = append(pub, "mesh.seat."+seat+".accept.>") + } + + // The two events it reacts to, and its ack subject on the stream they arrive from + // (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the + // controller a subscriber to every event in the mesh, and its permission list would stop + // saying what it is for. The ack grant below is scoped per stream because the controller's + // consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a + // delivery from EVENTS — a consumer that cannot ack has every message redelivered for + // ever, refused by the list it already has. + sub = append(sub, ControllerFollows...) + pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>") + + // **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That + // permits one reply to the reply subject of a message the user received — and a message a + // JetStream consumer delivers has had that field claimed for the consumer's own ack address + // (design 25 §2), so the address the controller actually answers is the one the request + // carried in its payload, which is not a reply subject as the server understands it. + // + // Verified against a real server before this line existed: the answer was refused with + // "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the + // mesh would have timed out while the controller logged success. + // + // **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and + // this is not it: nothing but an enrolling node ever subscribes under this prefix, each + // scoped to its own token's, so the controller publishing here is the mesh answering + // enrolments and can reach nothing else. + pub = append(pub, "_INBOX."+enrolmentPrefix+".>") + + case KindPerson: + // Tools, and nothing else. Every subject a person may publish is a tool call; a person + // who could publish an event would be able to claim a module said something. + for _, t := range p.Invokes { + if t == "*" { + pub = append(pub, "mesh.mod.*.tool.>") + continue + } + module, tool, ok := strings.Cut(t, ".") + if !ok { + return Permissions{}, fmt.Errorf( + "%q does not name a tool: a person invokes ., or * for every one", t) + } + pub = append(pub, "mesh.mod."+module+".tool."+tool) + } + + case KindEnrolment: + // A leaked token is useless for anything but enrolling: it cannot read a declaration, hear + // an event, or subscribe any inbox but the one its own token derives (design 25 §6). + // + // **The inbox was missing and the handshake could not have completed without it.** An + // enrolling node publishes its request and waits on an address it states in the payload; + // with nothing to subscribe it waits out its timeout against a mesh that answered. Its own + // and no wider: `_INBOX.enrol..>`, so what is sealed to one machine cannot be read by + // another enrolling beside it. + if p.Node == "" { + // Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty + // token in it — and worse, one every nameless enrolment user would share. A shared + // enrolment inbox is one machine able to read the credentials sealed to another. + return Permissions{}, errors.New( + "an enrolment user names no node, so its inbox would be shared with every other " + + "enrolment: a token is issued for a node record, and that record's name is " + + "the token's id") + } + pub = []string{"mesh.control.enrol"} + sub = []string{p.inbox()} + + case KindNode: + // A host publishes its own node's control traffic and subscribes its own declaration — + // and nothing of any other node's. + pub = []string{"mesh.control." + p.Node + ".>"} + sub = []string{"mesh.node." + p.Node + ".declare"} + + case KindModule: + // 1. Its own namespace: it publishes its events there and serves its tools there. Nothing + // else may publish into it, so an event's source is a fact the server enforces rather + // than a claim in the body (design 29 §2). + own := "mesh.mod." + p.Module + for _, e := range p.Emits { + pub = append(pub, own+".event."+e) + } + for _, t := range p.Serves { + sub = append(sub, own+".tool."+t) + } + + // 2. What it consumes, by the emitter's own subject — an event is addressed to its + // emitter, because the emitter's identity is the meaning (ADR 0118). + for _, c := range p.Consumes { + subject, err := consumedSubject(c) + if err != nil { + return Permissions{}, err + } + sub = append(sub, subject) + } + + // 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a + // role is hearing what it announced, not taking part in it. + for _, w := range p.Watches { + for _, e := range w.Emits { + sub = append(sub, seatSubject(w, "event", e)) + } + } + + // 3. Seats it holds: full participation. + for _, s := range p.Holds { + for _, a := range s.Accepts { + sub = append(sub, seatSubject(s, "accept", a)) + } + for _, e := range s.Emits { + pub = append(pub, seatSubject(s, "event", e)) + } + for _, t := range s.Serves { + sub = append(sub, seatSubject(s, "tool", t)) + } + } + + // 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a + // seat's inbound subject and watch other modules' traffic, nor publish its outbound + // events and lie about outcomes (design 29 §2). + for _, s := range p.Uses { + for _, a := range s.Accepts { + pub = append(pub, seatSubject(s, "accept", a)) + } + for _, t := range s.Serves { + pub = append(pub, seatSubject(s, "tool", t)) + } + } + } + + if p.Kind == KindPerson { + // An inbox to hear answers in, and nothing else. No ack subject: a person has no durable + // consumer, because nothing is delivered to a person — they ask and are answered. + sub = append(sub, p.inbox()) + } + + if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController { + // Its own reply space, and nothing wider. + sub = append(sub, p.inbox()) + + // Acking a JetStream delivery is a publish to that consumer's own ack address — a + // different subject from anything the consumer subscribes. Without it every message a + // module received would be redelivered forever, refused by the permission list it already + // has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own + // deliveries and no other's. + pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>") + } + + sort.Strings(pub) + sort.Strings(sub) + return Permissions{ + Publish: pub, + Subscribe: sub, + // Only something that serves is ever answering. A pure consumer is granted nothing here. + AllowResponses: p.Kind == KindModule && (len(p.Serves) > 0 || len(p.Holds) > 0) || + p.Kind == KindController, + }, nil +} + +// seatSubject places a seat's verb under the kind of traffic it is. +// +// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so +// without it a stream over a seat or a module's namespace would capture that namespace's *tool* +// traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS). +// Found while defining the streams: the first draft of design 29 had one namespace per module +// with no kind, which reads well and cannot be filtered. +// +// A seat serving more than its current protocol version carries the version as a token +// (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is +// bound to the old one. +func seatSubject(s Seat, kind, verb string) string { + return "mesh.seat." + s.Name + "." + kind + "." + verb +} + +// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are +// two functions because conflating them was a real bug. +// +// **A durable name may not contain a dot; an ack subject is built from two names that do.** The +// server acknowledges on `$JS.ACK...…`, so a single string "EVENTS.one_audit" +// reads correctly inside the permission and is rejected as a consumer name — *nats: invalid +// consumer name*. Caught against a running server, and worth the comment because the shape of +// the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack +// has every message redelivered forever, and its permission list looks right while it happens. +// +// They are derived here, beside the permission that must match them, because two places deriving +// the same name is how a module ends up unable to ack its own deliveries. +// consumedSubject is where a consumed event lands, from the local pattern a module declared. +// +// **The mesh's wildcards become this transport's** (design 29 §1): `*` is one name on both, and `**` +// — the rest — is `>` here. A module writes neither transport's spelling, so a manifest stays correct +// when the wire changes, which is the whole reason names are local. +// +// `**` on its own is every event from every module: the emitter is any, the event is anything. An +// audit logger wants exactly that and says so in one token. +func consumedSubject(pattern string) (string, error) { + if pattern == catalogueTheRest { + return "mesh.mod.*.event.>", nil + } + emitter, event, named := strings.Cut(pattern, ".") + if !named || emitter == "" || event == "" { + return "", fmt.Errorf( + "%q does not name an emitter and an event: a consumed event is ., or "+ + "%q for every event", pattern, catalogueTheRest) + } + if emitter == catalogueTheRest { + return "", fmt.Errorf("%q stands for the rest of a name, so it cannot name the emitter", catalogueTheRest) + } + // Each name is checked before it becomes a subject: a name carrying a dot would add a token and + // silently widen the permission, which is the whole reason safeSubject exists. + var out []string + for _, part := range strings.Split(event, ".") { + switch part { + case catalogueTheRest: + out = append(out, ">") + case "*": + out = append(out, "*") + default: + if !safeSubject.MatchString(part) { + return "", fmt.Errorf("%q cannot be part of a subject: it would widen the permission", part) + } + out = append(out, part) + } + } + if emitter != "*" && !safeSubject.MatchString(emitter) { + return "", fmt.Errorf("%q cannot name an emitter: it would widen the permission", emitter) + } + return "mesh.mod." + emitter + ".event." + strings.Join(out, "."), nil +} + +// catalogueTheRest is the mesh's wildcard for "the rest of a name", duplicated from the catalogue +// package for the one direction of dependency the build queue's name is duplicated for. +const catalogueTheRest = "**" + +func consumerStream(p Principal) string { + switch p.Kind { + case KindModule: + return "EVENTS" + case KindNode: + return "NODES" + case KindController: + return "CONTROL" + } + return "" +} + +func consumerDurable(p Principal) string { + switch p.Kind { + case KindModule: + return p.Node + "_" + p.Module + case KindNode: + return p.Node + case KindController: + return "controller" + } + return "" +} + +// Server is everything the composed file needs that is not a principal. +type Server struct { + // ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable + // without TLS is one a module can reach without TLS by mistake. + ClientPort int + MonitoringPort int + TLSCert string + TLSKey string + TLSCA string + // StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted + // four modules away from named volumes; the bus's own data is not the place to bring one back. + StoreDir string +} + +// Compose renders the server's whole configuration. The order is stable and the output is +// deterministic, because the file's digest is what the module's entrypoint watches to decide +// whether to reload: a composer that reordered a map on each run would signal a reload every time +// the controller restarted, for a file that had not changed. +func Compose(s Server, principals []Principal) (string, error) { + sorted := append([]Principal(nil), principals...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() }) + + var b strings.Builder + b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n") + b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n") + b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n") + + fmt.Fprintf(&b, "port: %d\n", s.ClientPort) + fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort) + + // **No `verify`, and it said `verify: true` until this configuration was run.** That setting + // makes the server demand a *client* certificate, and nothing in the mesh presents one: a host + // pins this server's exact certificate and authenticates with the password the mesh minted + // (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the + // mesh is refused at the TLS handshake before any password is looked at, and the error — + // "client didn't provide a certificate" — reads as a fault in the client. + // + // TLS is still required: the block is what requires it, and verify only decides whether client + // certificates are checked. + b.WriteString("tls {\n") + fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert) + fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey) + fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA) + b.WriteString("}\n\n") + + b.WriteString("jetstream {\n") + fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir) + b.WriteString("}\n\n") + + accounts, err := ComposeAccounts(sorted) + if err != nil { + return "", err + } + b.WriteString(accounts) + return b.String(), nil +} + +// ComposeAccounts is the accounts block alone — every user, and nothing about the server. +// +// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A +// server's ports, its TLS paths and its store directory are properties of the container the module +// raises: they live in its image and its mounts, and they change when it does. The controller has no +// business knowing them, and a controller that did would have to be kept in step with a Dockerfile +// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the +// module's own configuration includes it. +// +// Four things checked against a running server before this shape was committed to: a user in an +// included file authenticates; an unknown user is refused, so the include is the whole authority +// rather than an addition to something; a publish outside a user's grant is refused; and rewriting +// this file alone and signalling a reload makes a new user appear **without dropping the connection +// the mesh already has** — which is what makes every later account, permission or person change cost +// nothing (task 1.2's payoff). +func ComposeAccounts(principals []Principal) (string, error) { + sorted := append([]Principal(nil), principals...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() }) + + var b strings.Builder + b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n") + b.WriteString("# composition overwrites it. Permissions are derived from what each module\n") + b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n") + + // One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is + // one space (design 25 §4). The cost of that — that permissions are the only isolation — is + // paid in the scoping of every inbox and every ack subject. + b.WriteString("accounts {\n MESH {\n users = [\n") + for _, p := range sorted { + perms, err := PermissionsFor(p) + if err != nil { + return "", err + } + if p.PasswordHash == "" { + return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username()) + } + fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash) + fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish)) + fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe)) + if perms.AllowResponses { + b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n") + } + b.WriteString(" } }\n") + } + b.WriteString(" ]\n }\n}\n") + return b.String(), nil +} + +func quoted(values []string) string { + if len(values) == 0 { + return "" + } + out := make([]string, len(values)) + for i, v := range values { + out[i] = fmt.Sprintf("%q", v) + } + return strings.Join(out, ", ") +} diff --git a/internal/broker/nats_golden_test.go b/internal/broker/nats_golden_test.go new file mode 100644 index 0000000..6c3b2da --- /dev/null +++ b/internal/broker/nats_golden_test.go @@ -0,0 +1,49 @@ +package broker + +import ( + "flag" + "os" + "path/filepath" + "testing" +) + +var update = flag.Bool("update", false, "rewrite the golden composition") + +// The composed file is the mesh's whole authority model, so a change to it should be visible in a +// review rather than inferred from a diff of Go. The fixture is also the exact text checked +// against the real server's parser (`nats-server -t`), which is what says this syntax is the +// server's and not one we invented. +func TestTheComposedConfigMatchesTheGolden(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} + got, err := Compose( + Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data", + TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, + []Principal{ + {Kind: KindController, PasswordHash: "$2a$11$cccccccccccccccccccccc"}, + {Kind: KindEnrolment, Node: "one", PasswordHash: "$2a$11$eeeeeeeeeeeeeeeeeeeeee"}, + {Kind: KindNode, Node: "one", PasswordHash: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn"}, + {Kind: KindModule, Node: "one", Module: "telegram", Holds: []Seat{seat}, + Serves: []string{"status"}, PasswordHash: "$2a$11$tttttttttttttttttttttt"}, + {Kind: KindModule, Node: "two", Module: "shop", Uses: []Seat{seat}, + Emits: []string{"order.placed"}, PasswordHash: "$2a$11$ssssssssssssssssssssss"}, + {Kind: KindModule, Node: "two", Module: "audit", + Consumes: []string{"shop.order.placed"}, PasswordHash: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa"}, + }) + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "composed.conf") + if *update { + if err := os.WriteFile(golden, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if got != string(want) { + t.Errorf("composition changed; re-run with -update and read the diff:\n%s", got) + } +} diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go new file mode 100644 index 0000000..6ba56b1 --- /dev/null +++ b/internal/broker/nats_test.go @@ -0,0 +1,326 @@ +package broker + +import ( + "strings" + "testing" +) + +func has(t *testing.T, subjects []string, want string) { + t.Helper() + for _, s := range subjects { + if s == want { + return + } + } + t.Fatalf("expected %q among %v", want, subjects) +} + +func hasNot(t *testing.T, subjects []string, unwanted string) { + t.Helper() + for _, s := range subjects { + if s == unwanted { + t.Fatalf("did not expect %q among %v", unwanted, subjects) + } + } +} + +// A module's authority comes from its declaration and nothing else (novox/hq ADR 0043). +func TestAModulePublishesOnlyWhatItEmits(t *testing.T) { + p := Principal{Kind: KindModule, Node: "one", Module: "billing", + Emits: []string{"order.placed"}, PasswordHash: "x"} + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + has(t, perms.Publish, "mesh.mod.billing.event.order.placed") + hasNot(t, perms.Publish, "mesh.mod.billing.>") + hasNot(t, perms.Publish, "mesh.mod.shipping.event.order.placed") +} + +// The gap AMQP left open — an emitter granted the events exchange whole — is closed by per-subject +// permissions. A module cannot publish under another module's name. +func TestAModuleCannotPublishUnderAnothersName(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", + Emits: []string{"order.placed"}, PasswordHash: "x"}) + for _, p := range perms.Publish { + if strings.HasPrefix(p, "mesh.mod.") && !strings.HasPrefix(p, "mesh.mod.billing.") { + t.Fatalf("billing may publish %q, which is not its own namespace", p) + } + } +} + +// A caller of a seat may publish what the seat accepts, and nothing else of it: not its outbound +// events, and not a subscription to its inbound queue (design 29 §2). +func TestUsingASeatIsPublishOnlyAndInboundOnly(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "shop", + Uses: []Seat{seat}, PasswordHash: "x"}) + has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send") + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered") + hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send") +} + +// The holder is the mirror image: it consumes what the seat accepts and publishes what it emits. +func TestHoldingASeatIsTheMirrorOfUsingIt(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}} + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "telegram", + Holds: []Seat{seat}, PasswordHash: "x"}) + has(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send") + has(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered") + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.accept.send") +} + +// Without an ack permission a durable consumer never really consumes: every message it receives is +// redelivered forever, refused by the permission list it already has (design 25 §4). +func TestAModuleMayAckItsOwnDeliveriesAndNoOthers(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", + Consumes: []string{"shop.order.placed"}, PasswordHash: "x"}) + has(t, perms.Publish, "$JS.ACK.EVENTS.one_billing.>") + hasNot(t, perms.Publish, "$JS.ACK.>") + hasNot(t, perms.Publish, "$JS.ACK.EVENTS.one_shop.>") +} + +// With one account, inbox privacy is the permission list or it is nothing. +func TestAnInboxIsScopedToItsOwner(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", PasswordHash: "x"}) + has(t, perms.Subscribe, "_INBOX.one.billing.>") + hasNot(t, perms.Subscribe, "_INBOX.>") + hasNot(t, perms.Subscribe, "_INBOX.one.shop.>") +} + +// A responder answers on the caller's inbox, which it has no permission for. allow_responses is +// what makes a scoped inbox workable at all — the authority is bounded by having been asked. +func TestOnlySomethingThatServesMayAnswer(t *testing.T) { + serving, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "billing", + Serves: []string{"status"}, PasswordHash: "x"}) + if !serving.AllowResponses { + t.Fatal("a module serving a tool cannot answer the caller's inbox") + } + consumer, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "audit", + Consumes: []string{"shop.order.placed"}, PasswordHash: "x"}) + if consumer.AllowResponses { + t.Fatal("a pure consumer was granted the right to answer, which nothing asked it to do") + } +} + +// A host reaches its own node's control traffic and its own declaration, and nothing of any +// other node's. +func TestAHostIsConfinedToItsOwnNode(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindNode, Node: "one", PasswordHash: "x"}) + has(t, perms.Publish, "mesh.control.one.>") + has(t, perms.Subscribe, "mesh.node.one.declare") + hasNot(t, perms.Subscribe, "mesh.node.two.declare") + hasNot(t, perms.Subscribe, "mesh.node.>") +} + +// A leaked enrolment token is useless for anything but enrolling (design 25 §6). +func TestTheEnrolmentUserCanOnlyEnrol(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.control.enrol" { + t.Fatalf("enrolment may publish %v", perms.Publish) + } + // Its own inbox and nothing else. **Nothing else** is the point: no declaration, no event, and + // no other machine's answer — and the inbox itself is needed, because a node that cannot + // subscribe one waits out its timeout against a mesh that answered. + if len(perms.Subscribe) != 1 || perms.Subscribe[0] != "_INBOX.enrol.anchor.>" { + t.Fatalf("enrolment may subscribe %v, which is not its own inbox alone", perms.Subscribe) + } +} + +// An enrolment user that names no node is refused: its inbox would be an empty subject token, and +// one that every nameless enrolment user shared — which is one machine reading the credentials +// sealed to another. +func TestAnEnrolmentUserWithoutANodeIsRefused(t *testing.T) { + if _, err := PermissionsFor(Principal{Kind: KindEnrolment, PasswordHash: "x"}); err == nil { + t.Fatal("an enrolment user with no node was composed, so its inbox is shared") + } +} + +// A name that would widen a permission is refused rather than quietly stretching one. +func TestANameThatWouldWidenAPermissionIsRefused(t *testing.T) { + for _, bad := range []string{"bill.ing", "billing.>", "*", "bil>ling"} { + if _, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: bad, PasswordHash: "x"}); err == nil { + t.Fatalf("%q was accepted as part of a subject", bad) + } + } +} + +// The entrypoint reloads on the file's digest changing, so an unchanged mesh must compose an +// identical file — otherwise every controller restart signals a reload of the whole bus. +func TestComposingTwiceGivesTheSameBytes(t *testing.T) { + s := Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data", + TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"} + ps := []Principal{ + {Kind: KindModule, Node: "two", Module: "shop", Emits: []string{"order.placed"}, PasswordHash: "b"}, + {Kind: KindController, PasswordHash: "c"}, + {Kind: KindModule, Node: "one", Module: "billing", Consumes: []string{"shop.order.placed"}, PasswordHash: "a"}, + } + first, err := Compose(s, ps) + if err != nil { + t.Fatal(err) + } + shuffled := []Principal{ps[2], ps[0], ps[1]} + second, err := Compose(s, shuffled) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatal("composition is order-dependent; every controller restart would reload the bus") + } +} + +// A user without a password is a user anybody is. +func TestAUserWithoutAPasswordIsRefused(t *testing.T) { + _, err := Compose(Server{ClientPort: 4222}, []Principal{{Kind: KindController}}) + if err == nil { + t.Fatal("composed a user with no password hash") + } +} + +// A person reaches the mesh's tools from a workstation (design 25 §7). Their authority is a list +// of tools and nothing else. +func TestAPersonMayAskOnlyTheToolsTheyWereGiven(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"shop.price", "telegram.status"}, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + has(t, perms.Publish, "mesh.mod.shop.tool.price") + has(t, perms.Publish, "mesh.mod.telegram.tool.status") + hasNot(t, perms.Publish, "mesh.mod.shop.tool.refund") + hasNot(t, perms.Publish, "mesh.mod.*.tool.>") +} + +// An administrator gets every tool, which is a different grant and looks like one. +func TestAnAdministratorMayAskAnyTool(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + has(t, perms.Publish, "mesh.mod.*.tool.>") +} + +// **Nothing but tools.** A person who could publish an event would be able to claim a module +// said something; one who could publish control traffic would be a second controller. +func TestAPersonReachesNothingButTools(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + for _, p := range perms.Publish { + if !strings.Contains(p, ".tool.") { + t.Errorf("a person may publish %q, which is not a tool call", p) + } + } + for _, s := range perms.Subscribe { + if !strings.HasPrefix(s, "_INBOX.person.") { + t.Errorf("a person may subscribe %q; only their own inbox should be reachable", s) + } + } +} + +// A person has no durable consumer, because nothing is delivered to a person — so no ack +// subject, and an ack permission would be authority over something that does not exist. +func TestAPersonHasNoAckSubject(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + for _, p := range perms.Publish { + if strings.HasPrefix(p, "$JS.ACK") { + t.Errorf("a person was granted %q, and has no consumer to acknowledge", p) + } + } +} + +// A person asks and is answered; they never answer. allow_responses would let a person reply to +// a request — which, on a bus where anyone may serve a tool, is somebody impersonating a module. +func TestAPersonMayNotAnswer(t *testing.T) { + perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"*"}, PasswordHash: "x"}) + if perms.AllowResponses { + t.Fatal("a person may answer a request, which is impersonating a module") + } +} + +// Two people do not share an inbox, or one would read the other's answers. +func TestTwoPeopleDoNotShareAnInbox(t *testing.T) { + a, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"}) + b, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "sam", Invokes: []string{"*"}, PasswordHash: "x"}) + if a.Subscribe[0] == b.Subscribe[0] { + t.Fatalf("both read %s", a.Subscribe[0]) + } +} + +// A malformed grant is refused rather than widened into something that happens to parse. +func TestAToolGrantThatNamesNoToolIsRefused(t *testing.T) { + if _, err := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", + Invokes: []string{"shop"}, PasswordHash: "x"}); err == nil { + t.Fatal("a grant naming a module but no tool was accepted") + } +} + +// The controller can answer an enrolment, and reach no other inbox. +// +// **`allow_responses` does not cover this and that is the trap.** It permits one reply to the reply +// subject of a message the user received — and a message a JetStream consumer delivers has had that +// field claimed for the consumer's own ack address, so the address the controller actually answers is +// the one the request carried in its payload, which the server does not recognise as a reply subject +// at all. +// +// Found against a real server, after a live test on an *unpermissioned* one had passed: every +// enrolment on the mesh would have timed out while the controller logged success. +func TestTheControllerCanAnswerAnEnrolmentAndReachNoOtherInbox(t *testing.T) { + ctl, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + enrolling, err := PermissionsFor(Principal{Kind: KindEnrolment, Node: "anchor", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + + // Whatever the enrolling node waits on, the controller must be able to publish to. + if len(enrolling.Subscribe) != 1 { + t.Fatalf("an enrolling node subscribes %v, and this test knows only how to check one", + enrolling.Subscribe) + } + waitsOn := enrolling.Subscribe[0] + if !covers(ctl.Publish, waitsOn) { + t.Fatalf("the controller may publish %v, none of which reaches %s — so every enrolment on "+ + "the mesh times out while the controller logs success", ctl.Publish, waitsOn) + } + + // And nothing wider. A node's own inbox and a module's are not the controller's to write into: + // that is the blanket grant design 25 §4 refuses. + for _, other := range []string{"_INBOX.node.anchor.x", "_INBOX.one.shop.x", "_INBOX.person.ada.x"} { + if covers(ctl.Publish, other) { + t.Errorf("the controller can publish to %s, which is an inbox privacy the permission "+ + "list is the only thing protecting", other) + } + } +} + +// covers says whether any granted subject pattern admits one concrete subject, with NATS's own +// wildcard meanings: `*` is one token, `>` is the rest. +func covers(granted []string, subject string) bool { + want := strings.Split(subject, ".") + for _, pattern := range granted { + if admits(strings.Split(pattern, "."), want) { + return true + } + } + return false +} + +func admits(pattern, subject []string) bool { + for i, token := range pattern { + if token == ">" { + return i < len(subject) + } + if i >= len(subject) { + return false + } + if token != "*" && token != subject[i] { + return false + } + } + return len(pattern) == len(subject) +} diff --git a/internal/broker/onnats.go b/internal/broker/onnats.go new file mode 100644 index 0000000..245f429 --- /dev/null +++ b/internal/broker/onnats.go @@ -0,0 +1,91 @@ +package broker + +import ( + "fmt" + "strings" + + "github.com/novox/mesh-controller/internal/envfile" +) + +// Whether this mesh's own traffic is on the bus being built. +// +// **One switch, read in one place** (novox/hq ADR 0116 step 5). Every seam the bus change went +// behind ships both implementations, and until the rollout every one of them chooses the bus the +// mesh runs on today. This is what the rollout flips, and it is deliberately a single fact rather +// than a fact per component: a controller whose outbound is on one bus and whose inbound is on the +// other is a mesh that hears nothing, and no test of either half would catch it. + +// NATSVar is where the controller finds the bus being built. Unset is the ordinary case and means +// the mesh runs on the bus it has always run on. +const NATSVar = "MESH_BUS_NATS" + +// OnNATS is the address of the bus being built, and whether the mesh is on it. +// +// Read from the node's own settings rather than baked in, for the reason the broker's address is +// (novox/hq 04-ISSUES/102): an address recorded once does not follow a node's ports. +func OnNATS() (address string, on bool, err error) { + address, err = envfile.Placed(NATSVar) + if err != nil { + return "", false, err + } + address = strings.TrimSpace(address) + if address == "" { + return "", false, nil + } + return address, true, nil +} + +// CredentialIn reads the user and password out of a bus address, and the address without them. +// +// The controller's own credential arrives in its address, the way the old bus's does. Split out so the +// controller can record a hash of what it is actually using: its user is created by the installer at a +// bootstrap password, before the controller exists to mint one, and a composition that left itself out +// would produce a bus the writer cannot connect to. +func CredentialIn(address string) (user, password, bare string) { + at := strings.LastIndex(address, "@") + if at < 0 { + return "", "", address + } + scheme := "" + rest := address[:at] + if i := strings.Index(rest, "://"); i >= 0 { + scheme, rest = rest[:i+3], rest[i+3:] + } + user, password, _ = strings.Cut(rest, ":") + return user, password, scheme + address[at+1:] +} + +// BareAddress is a bus address with any credential stripped, for something that only needs to know +// whether a server is answering there. +func BareAddress(address string) string { + _, _, bare := CredentialIn(address) + if bare == "" { + return address + } + if strings.Contains(bare, "://") { + return bare + } + return "nats://" + bare +} + +// MustBeOneBus refuses a configuration that names both buses for the mesh's own traffic. +// +// **Both clients ship and that is the point; both being live is not.** The rollout moves every node +// at once (ADR 0116 step 5): a mesh half on each is one where a declaration goes out on one bus and +// the report comes back on the other, and nothing anywhere says so — every component would log +// success. Refused at start, where it can be said in one sentence. +func MustBeOneBus(amqp, nats string) error { + if strings.TrimSpace(amqp) != "" && strings.TrimSpace(nats) != "" { + return fmt.Errorf( + "this control plane is told about both buses (%s and %s) and can only be on one. A mesh "+ + "half on each is one where a declaration goes out on one and the report comes back "+ + "on the other, and every component reports success while it happens. The rollout "+ + "moves every node at once: unset %s to stay, or unset %s to move", + AMQPVarName, NATSVar, NATSVar, AMQPVarName) + } + return nil +} + +// AMQPVarName is the variable naming the bus the mesh runs on today. Named here rather than +// imported from the link package, for the one direction of dependency. +const AMQPVarName = "MESH_BROKER_AMQP" diff --git a/internal/broker/onnats_test.go b/internal/broker/onnats_test.go new file mode 100644 index 0000000..f21bc87 --- /dev/null +++ b/internal/broker/onnats_test.go @@ -0,0 +1,60 @@ +package broker + +import ( + "strings" + "testing" +) + +// Which bus the mesh is on is one fact, and being told about both is refused. +// +// **Not a warning.** A mesh half on each bus is one where a declaration goes out on one and the +// report comes back on the other, and every component reports success while it happens — which is +// the exact failure ADR 0074 exists to catch, arriving through configuration instead of through code. +func TestBeingToldAboutBothBusesIsRefused(t *testing.T) { + err := MustBeOneBus("amqps://broker:5671/", "nats://bus:4222") + if err == nil { + t.Fatal("a control plane told about both buses was allowed to start") + } + // The remedy is in the words, because whoever reads this has to choose one and the wrong choice + // is a rollout half done. + for _, want := range []string{AMQPVarName, NATSVar, "unset"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not mention %s: %v", want, err) + } + } +} + +// One bus, or none, is ordinary. None is a control plane that publishes nothing and holds records, +// which several of its own commands are. +func TestOneBusOrNeitherIsAllowed(t *testing.T) { + for _, c := range []struct{ what, amqp, nats string }{ + {"the bus the mesh runs on today", "amqps://broker:5671/", ""}, + {"the bus being built", "", "nats://bus:4222"}, + {"neither", "", ""}, + {"neither, with whitespace for an address", " ", "\t"}, + } { + if err := MustBeOneBus(c.amqp, c.nats); err != nil { + t.Errorf("%s was refused: %v", c.what, err) + } + } +} + +// The controller's own credential arrives in its address, and has to be readable out of it — its user +// is created by the installer at a bootstrap password, before the controller exists to mint one. +func TestACredentialIsReadOutOfABusAddress(t *testing.T) { + for _, c := range []struct{ in, user, password, bare string }{ + {"nats://controller:secret@127.0.0.1:4222", "controller", "secret", "nats://127.0.0.1:4222"}, + {"controller:secret@127.0.0.1:4222", "controller", "secret", "127.0.0.1:4222"}, + {"nats://127.0.0.1:4222", "", "", "nats://127.0.0.1:4222"}, + {"127.0.0.1:4222", "", "", "127.0.0.1:4222"}, + // A password containing an at-sign: split on the last one, or the address becomes part of the + // credential and the connection goes somewhere nobody named. + {"nats://controller:a@b@127.0.0.1:4222", "controller", "a@b", "nats://127.0.0.1:4222"}, + } { + user, password, bare := CredentialIn(c.in) + if user != c.user || password != c.password || bare != c.bare { + t.Errorf("%q read as %q/%q at %q; wanted %q/%q at %q", + c.in, user, password, bare, c.user, c.password, c.bare) + } + } +} diff --git a/internal/broker/raise.go b/internal/broker/raise.go new file mode 100644 index 0000000..6f3d15a --- /dev/null +++ b/internal/broker/raise.go @@ -0,0 +1,73 @@ +package broker + +import "fmt" + +// Bringing the bus's own objects into being, in the one order that works. +// +// **Asserted on every start rather than created once at genesis.** A stream somebody deleted, a mesh +// raised from a restored backup, or a bus whose data directory was replaced all have records and no +// objects — and a node whose consumer is missing hears nothing while everything else about it looks +// correct. Idempotence is the whole requirement, and the parts are already idempotent; this is the +// order they have to be asked in. + +// Raiser is everything asserting the bus's objects needs of a connection to it. +type Raiser interface { + Asserter + Ensurer +} + +// Raise asserts the mesh's streams, the controller's own consumers, and one consumer per node. +// +// **The order is not a preference.** A consumer on a stream that does not exist is refused, and the +// refusal names the stream rather than the order — so somebody reading it goes looking for a deleted +// stream instead of a reversed pair of lines. Nodes last, because the one a node reads lives on a +// stream the mesh's own set defines. +func Raise(r Raiser, nodes []string) error { + if err := AssertMeshStreams(r); err != nil { + return err + } + if err := AssertMeshConsumers(r); err != nil { + return err + } + if err := AssertNodeConsumers(r, nodes); err != nil { + return err + } + return nil +} + +// RaiseSeats asserts one work queue per declared seat, and the worker of whoever holds it. +// +// Separate from Raise because it is answered by a different question: the mesh's own objects exist +// because the mesh does, and a seat's exist because a module declaring one was registered. Kept +// beside it so the order is visible — a holder's worker needs the seat's stream, and a seat's stream +// needs nothing. +func RaiseSeats(r Raiser, seats []DeclaredSeat, holders map[string]Holder) error { + for _, s := range SeatStreams(seats) { + if err := r.EnsureStream(s); err != nil { + return fmt.Errorf("asserting the work queue for %s: %w", s.Name, err) + } + } + for _, s := range seats { + h, held := holders[s.Name] + if !held { + // **The stream exists and the consumer does not, on purpose.** Work queues until a + // holder appears, so installing the module a week after something started sending to it + // flushes the backlog instead of having lost it. + continue + } + c, needed := HolderConsumerFor(h.Node, h.Module, s) + if !needed { + continue + } + if err := r.EnsureConsumer(c); err != nil { + return fmt.Errorf("asserting how %s on %s works %s: %w", h.Module, h.Node, s.Name, err) + } + } + return nil +} + +// Holder is which module on which machine holds a seat. +type Holder struct { + Node string + Module string +} diff --git a/internal/broker/raise_live_test.go b/internal/broker/raise_live_test.go new file mode 100644 index 0000000..202babc --- /dev/null +++ b/internal/broker/raise_live_test.go @@ -0,0 +1,196 @@ +package broker + +import ( + "os" + "testing" + + "github.com/nats-io/nats.go" +) + +// Raising the bus's objects against a real server. +// +// The pure tests above say what is asked for and in what order. Only a server can say whether it +// accepts them — and two of these are claims about the server's own behaviour that nothing else +// could answer: that asserting twice changes nothing, and that a consumer really is bound to the one +// subject its node is allowed to read. +// +// docker run -d --rm --name t -p 14227:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14227 go test ./internal/broker/ -run TestRaising + +func aLiveBus(t *testing.T) *JetStream { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + // **Nothing is deleted here, deliberately.** These objects are the mesh's own and every live + // test in every package shares one server: a test that deleted a stream to get a clean slate + // took it out from under whatever was running beside it, and the failure landed in the other + // test as "stream not found" — which reads as a bug in the code under test. Raise is idempotent + // by requirement, so asserting against whatever is already there is both safe and the realistic + // case. + return js +} + +// Every object the mesh's own traffic needs, accepted by a real server, and asserting again changes +// nothing — which is the whole requirement, because this runs on every start. +func TestRaisingTheBusIsAcceptedAndIdempotent(t *testing.T) { + js := aLiveBus(t) + + if err := Raise(js, []string{"anchor", "laptop"}); err != nil { + t.Fatalf("a real server refused the mesh's own objects: %v", err) + } + // Twice, with nothing in between. A start that failed the second time is a controller that + // cannot restart. + if err := Raise(js, []string{"anchor", "laptop"}); err != nil { + t.Fatalf("asserting the bus's objects a second time failed, so a restart would: %v", err) + } + // And again with a machine that was not there before, which is what enrolling one is. + if err := Raise(js, []string{"anchor", "laptop", "workstation"}); err != nil { + t.Fatalf("a machine joining an already-raised bus was refused: %v", err) + } + + for _, s := range MeshStreams() { + if _, err := js.Context().StreamInfo(s.Name); err != nil { + t.Errorf("stream %s is not there: %v", s.Name, err) + } + } + for _, c := range MeshConsumers() { + if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil { + t.Errorf("the controller's consumer on %s is not there: %v", c.Stream, err) + } + } + for _, node := range []string{"anchor", "laptop", "workstation"} { + info, err := js.Context().ConsumerInfo("NODES", node) + if err != nil { + t.Errorf("%s has no way to hear its declaration: %v", node, err) + continue + } + // **Its own subject and no other node's.** A consumer filtered on anything wider is a node + // reading another machine's declaration, and its own ack grant would not cover it either. + if info.Config.FilterSubject != "mesh.node."+node+".declare" { + t.Errorf("%s's consumer reads %q", node, info.Config.FilterSubject) + } + if info.Config.AckPolicy != nats.AckExplicitPolicy { + t.Errorf("%s's consumer acknowledges on delivery, so a declaration it died applying is "+ + "never sent again", node) + } + } +} + +// The store window needs unlimited redelivery on CONTROL: the bound belongs to the controller, and a +// server that dead-lettered first would discard the push the stream exists to protect. +func TestTheControlConsumerDoesNotDeadLetterBeforeTheControllerGivesUp(t *testing.T) { + js := aLiveBus(t) + if err := Raise(js, nil); err != nil { + t.Fatal(err) + } + info, err := js.Context().ConsumerInfo("CONTROL", ControllerName) + if err != nil { + t.Fatal(err) + } + if info.Config.MaxDeliver > 0 { + t.Fatalf("max-deliver is %d: a push held through a store restart would be dead-lettered "+ + "before the controller finished deciding about it", info.Config.MaxDeliver) + } +} + +// A role's work queue exists before anybody holds it, against a real server. +// +// **The queue before the holder is the point** (novox/hq ADR 0121): work queues until somebody arrives +// to do it, so assigning a build machine a week after something started asking for builds flushes the +// backlog instead of having lost it. A stream created at assignment would make "the holder is not here +// yet" mean "your requests are gone". +func TestRaisingAMeshRolesWorkQueue(t *testing.T) { + js := aLiveBus(t) + seats := []DeclaredSeat{{Name: "mesh-build-machine", Accepts: []string{"build"}, + Emits: []string{"built"}}} + t.Cleanup(func() { _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") }) + + if err := RaiseSeats(js, seats, nil); err != nil { + t.Fatalf("a real server refused a role's work queue: %v", err) + } + info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE") + if err != nil { + t.Fatalf("the role has no work queue: %v", err) + } + if info.Config.Retention != nats.WorkQueuePolicy { + t.Errorf("the queue retains as %v: work a holder took must leave it, or the next holder does "+ + "it again", info.Config.Retention) + } + if len(info.Config.Subjects) != 1 || info.Config.Subjects[0] != "mesh.seat.mesh-build-machine.accept.>" { + t.Errorf("it carries %v rather than the role's own inbound subjects", info.Config.Subjects) + } + // Nobody holds it, so there is no worker — and asserting again changes nothing, because this runs + // on every start. + if err := RaiseSeats(js, seats, nil); err != nil { + t.Fatalf("asserting a role's queue a second time failed, so a restart would: %v", err) + } + + // And once somebody holds it, the worker appears on that same queue. + if err := RaiseSeats(js, seats, map[string]Holder{ + "mesh-build-machine": {Node: "anchor", Module: "builder"}, + }); err != nil { + t.Fatal(err) + } + if _, err := js.Context().ConsumerInfo("SEAT_MESH_BUILD_MACHINE", + "SEAT_MESH_BUILD_MACHINE_worker"); err != nil { + t.Fatalf("the holder got no worker on the role's queue: %v", err) + } +} + +// **A consumer created after the fact still sees what came before it**, which is why the mesh needs no +// catch-up at all on this bus (novox/hq 04-ISSUES/050). +// +// On the bus the mesh runs on today a queue receives only what is published after it is bound, so +// everything built before the catalogue existed was announced to nobody — and on a fresh mesh that is +// always the foundation, because those are the things the catalogue needed in order to exist. A whole +// mechanism was built for it: the catalogue asks, the controller re-publishes. +// +// A stream is a log and a consumer is a position in it. A consumer created later starts at the +// beginning by default, so the builds are simply there. Asked of a real server rather than assumed, +// because the whole decision about whether to keep that mechanism rests on it. +func TestAConsumerCreatedAfterwardsStillSeesWhatCameBefore(t *testing.T) { + js := aLiveBus(t) + if err := AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.Context().PurgeStream("EVENTS"); err != nil { + t.Fatal(err) + } + + // Genesis: things are built before anything is listening. + built := []string{"base", "store", "mesh-catalog"} + for _, m := range built { + if _, err := js.Context().Publish("mesh.seat.mesh-build-machine.event.built", + []byte(`{"module":"`+m+`"}`)); err != nil { + t.Fatal(err) + } + } + + // Now the catalogue is installed and the controller creates its consumer. + c, ok := ConsumerFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-catalog", + Watches: []Seat{{Name: "mesh-build-machine", Emits: []string{"built"}}}, PasswordHash: "x"}) + if !ok { + t.Fatal("a module that watches a role got no consumer") + } + t.Cleanup(func() { _ = js.Context().DeleteConsumer(c.Stream, c.Name) }) + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + + info, err := js.Context().ConsumerInfo(c.Stream, c.Name) + if err != nil { + t.Fatal(err) + } + if info.NumPending != uint64(len(built)) { + t.Fatalf("a consumer created after %d builds has %d waiting for it — if this is 0 the mesh "+ + "does need a catch-up after all, and the reasoning for deleting it is wrong", + len(built), info.NumPending) + } +} diff --git a/internal/broker/readiness.go b/internal/broker/readiness.go new file mode 100644 index 0000000..a1302d3 --- /dev/null +++ b/internal/broker/readiness.go @@ -0,0 +1,142 @@ +package broker + +import ( + "fmt" + "sort" + "strings" +) + +// Whether a mesh could move its bus, and what is missing if not. +// +// **Asked before anything moves, and answerable from records alone.** The rollout moves every node at +// once (novox/hq ADR 0116 step 5), so there is no partial state to inspect afterwards and no half to +// roll back: either the mesh was ready or it was not. That makes a readiness question the most +// valuable thing here — it costs nothing, it can be asked of a running mesh any number of times, and +// every answer is a thing somebody can go and fix. +// +// Deliberately pure. It is handed what the mesh knows and returns sentences; nothing here connects to +// anything, so it can be asked on a workstation about a mesh it has never reached. + +// Readiness is what the mesh knows about its own ability to move. +type Readiness struct { + // TheBus is the address the mesh's own traffic would move to, empty when nothing names one. + TheBus string + // ServerStanding is whether a bus is reachable at that address, as somebody checked. + ServerStanding bool + // Holder is the node running the module that holds the bus seat, empty when nothing does. + Holder string + // AccountsComposed is whether that node has been sent the composed user list. + AccountsComposed bool + // Nodes is every machine the mesh knows. + Nodes []string + // Credentialled is which of them has a credential for the new bus. + Credentialled map[string]bool + // Modules is every assigned module, as `/`. + Modules []string + // ModuleCredentialled is which of those has one. + ModuleCredentialled map[string]bool + // StillOnTheOldBus is whether anything of the mesh's own still needs the bus it is leaving — + // which is not a reason to stop, because that broker stays as an ordinary provider of `amqp` + // (ADR 0119). Recorded so nobody reads the move as a retirement. + OldBusHasOtherClients bool +} + +// NotReady is every reason this mesh cannot move its bus yet, in the order somebody would fix them. +// +// Empty means ready. **Each entry names one thing and what to do about it**, because a readiness check +// that says "not ready" is a check nobody can act on — and this is read at the point where the next +// step is irreversible. +func NotReady(r Readiness) []string { + var why []string + + if strings.TrimSpace(r.TheBus) == "" { + why = append(why, "nothing names the bus to move to: set "+NATSVar+" on the control node "+ + "to the address the new server answers on") + } + if !r.ServerStanding { + why = append(why, "no bus is answering at that address. Step 2 of the change raises it beside "+ + "the one the mesh is on, carrying nothing — assign the module that holds "+ + "mesh-broker and push the machine that runs it") + } + if r.Holder == "" { + why = append(why, "no machine holds mesh-broker, so nothing would compose the bus's user "+ + "list. Assign the module that claims it") + } else if !r.AccountsComposed { + why = append(why, fmt.Sprintf( + "%s holds mesh-broker and has not been sent the composed user list, so the bus would "+ + "refuse every connection. `push %s`", r.Holder, r.Holder)) + } + + // A node with no credential cannot come back after the move, and a node that cannot come back is + // a machine the mesh has lost until somebody goes to it. + var missing []string + for _, n := range r.Nodes { + if !r.Credentialled[n] { + missing = append(missing, n) + } + } + sort.Strings(missing) + if len(missing) > 0 { + why = append(why, fmt.Sprintf( + "%d machine(s) have no credential for the new bus and would not come back: %s. Each needs "+ + "one minted before the move, not after — after, there is no bus to ask over", + len(missing), strings.Join(missing, ", "))) + } + + // A module without one keeps running and stops being reachable, which is a smaller fault and still + // one somebody should choose rather than discover. + var quiet []string + for _, m := range r.Modules { + if !r.ModuleCredentialled[m] { + quiet = append(quiet, m) + } + } + sort.Strings(quiet) + if len(quiet) > 0 { + why = append(why, fmt.Sprintf( + "%d module(s) have no credential for the new bus: %s. Each keeps serving and stops "+ + "answering tools and hearing events until it is issued one", + len(quiet), strings.Join(quiet, ", "))) + } + + return why +} + +// WhatMoves is what the rollout would do, in order, for somebody reading before they commit. +// +// **Written out rather than summarised.** This is the one step with nothing to inspect afterwards, so +// the last useful moment to disagree with it is while reading this. +func WhatMoves(r Readiness) []string { + out := []string{ + fmt.Sprintf("compose the bus's user list and send it to %s", holderOr(r.Holder)), + fmt.Sprintf("move this control plane to %s, and confirm it is heard", busOr(r.TheBus)), + } + nodes := append([]string(nil), r.Nodes...) + sort.Strings(nodes) + for _, n := range nodes { + out = append(out, fmt.Sprintf("move %s, and confirm it reports", n)) + } + if len(r.Modules) > 0 { + out = append(out, fmt.Sprintf("move %d module runtime(s), and confirm each answers", + len(r.Modules))) + } + if r.OldBusHasOtherClients { + out = append(out, "leave the old broker running: it stays an ordinary provider of `amqp` for "+ + "whatever else uses it (ADR 0119), and this move is not its retirement") + } + return out +} + +func holderOr(node string) string { + if node == "" { + return "whichever machine holds mesh-broker" + } + return node +} + +func busOr(address string) string { + if address == "" { + return "the new bus" + } + return address +} diff --git a/internal/broker/readiness_test.go b/internal/broker/readiness_test.go new file mode 100644 index 0000000..827b786 --- /dev/null +++ b/internal/broker/readiness_test.go @@ -0,0 +1,97 @@ +package broker + +import ( + "strings" + "testing" +) + +// Whether a mesh could move its bus. +// +// Every case here is a way of moving that leaves something behind, and the one that matters most is a +// machine with no credential: after the move there is no bus to ask it over, so it is lost until +// somebody walks to it. + +func aMeshReadyToMove() Readiness { + return Readiness{ + TheBus: "nats://127.0.0.1:5671", ServerStanding: true, + Holder: "anchor", AccountsComposed: true, + Nodes: []string{"anchor", "laptop"}, + Credentialled: map[string]bool{"anchor": true, "laptop": true}, + Modules: []string{"anchor/gitea"}, + ModuleCredentialled: map[string]bool{"anchor/gitea": true}, + } +} + +func TestAMeshWithEverythingInPlaceIsReady(t *testing.T) { + if why := NotReady(aMeshReadyToMove()); len(why) != 0 { + t.Fatalf("a mesh with everything in place was refused: %v", why) + } +} + +// **A machine with no credential is the one that must stop this.** It keeps running and cannot come +// back, and there is no bus left to tell it anything over — so the remedy has to happen before, and +// the message says so. +func TestAMachineWithNoCredentialStopsTheMove(t *testing.T) { + r := aMeshReadyToMove() + r.Credentialled = map[string]bool{"anchor": true} + + why := NotReady(r) + if len(why) == 0 { + t.Fatal("a machine that could not come back did not stop the move") + } + said := strings.Join(why, "\n") + if !strings.Contains(said, "laptop") { + t.Errorf("the refusal does not name the machine: %s", said) + } + if !strings.Contains(said, "before the move") { + t.Errorf("the refusal does not say the remedy comes first: %s", said) + } +} + +// A bus nobody has raised, a seat nobody holds, and a user list nobody has been sent: each stops it, +// and each names its own next step, because "not ready" that cannot be acted on is not an answer. +func TestEachThingMissingNamesItsOwnRemedy(t *testing.T) { + for _, c := range []struct { + what string + break_ func(*Readiness) + says string + }{ + {"no address", func(r *Readiness) { r.TheBus = "" }, NATSVar}, + {"no server", func(r *Readiness) { r.ServerStanding = false }, "carrying nothing"}, + {"no holder", func(r *Readiness) { r.Holder = "" }, "mesh-broker"}, + {"no user list", func(r *Readiness) { r.AccountsComposed = false }, "push anchor"}, + {"a module with none", func(r *Readiness) { + r.ModuleCredentialled = map[string]bool{} + }, "anchor/gitea"}, + } { + r := aMeshReadyToMove() + c.break_(&r) + why := NotReady(r) + if len(why) == 0 { + t.Errorf("%s did not stop the move", c.what) + continue + } + if !strings.Contains(strings.Join(why, "\n"), c.says) { + t.Errorf("%s: the refusal does not mention %q: %v", c.what, c.says, why) + } + } +} + +// What the move would do is written out rather than summarised, because this is the one step with +// nothing to inspect afterwards — so reading it is the last chance to disagree. +func TestWhatMovesNamesEveryMachineAndSaysTheOldBrokerStays(t *testing.T) { + r := aMeshReadyToMove() + r.OldBusHasOtherClients = true + steps := strings.Join(WhatMoves(r), "\n") + + for _, want := range []string{"anchor", "laptop", "user list", "module runtime"} { + if !strings.Contains(steps, want) { + t.Errorf("the plan does not mention %q:\n%s", want, steps) + } + } + // Said explicitly, so nobody reads the move as switching the old broker off — it stays serving + // whatever else uses it, and that is a decision already taken. + if !strings.Contains(steps, "not its retirement") { + t.Errorf("the plan does not say the old broker stays:\n%s", steps) + } +} diff --git a/internal/broker/streams.go b/internal/broker/streams.go new file mode 100644 index 0000000..9b31266 --- /dev/null +++ b/internal/broker/streams.go @@ -0,0 +1,238 @@ +package broker + +import ( + "fmt" + "sort" +) + +// The mesh's own streams. +// +// **These four and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118). An earlier +// reading had the controller create *every* stream at genesis, from a fixed set. That is only the +// mesh's own half: a seat's streams are created when the module declaring it is registered, and a +// module's durable consumers when it is assigned — neither of which has happened at genesis. What +// is here is the foundation, which exists before any module does. +// +// The controller is the only writer of stream definitions (design 25 §3). A module declares +// nothing about them and cannot reach the JetStream API to make one. + +// Retention is how a stream decides what to keep, which is the whole of what distinguishes the +// mesh's four relationships on the wire (design 29 §4). +type Retention string + +const ( + // RetentionWorkQueue: a message is removed once a consumer acknowledges it. Exactly one + // worker does the work, and a worker that dies has its message redelivered. + RetentionWorkQueue Retention = "workqueue" + // RetentionLastPerSubject: only the newest message on each subject survives. This is the + // state shape — a node that was away gets exactly the current declaration and nothing older. + RetentionLastPerSubject Retention = "last_per_subject" + // RetentionLimits: kept until it ages or the stream fills. Events, where a subscriber that + // was down catches up and nobody is obliged to act. + RetentionLimits Retention = "limits" +) + +// A Stream is one of the mesh's own, as the controller asserts it. +type Stream struct { + Name string + Subjects []string + Retention Retention + // MaxAge in seconds, zero for unbounded. Per stream — JetStream has no per-subject age, + // which is why differing retention between modules would mean a stream each. + MaxAge int + // MaxMsgsPerSubject caps each subject independently, so one noisy emitter cannot push + // another's events out of a shared stream. Verified: with a cap of 3, ten messages on one + // subject and one on another leave four in the stream, not three. + MaxMsgsPerSubject int + // Why is carried into the assertion so an operator reading the server's own state finds the + // reason there, rather than only in a repository they may not have. + Why string +} + +// MeshStreams is the foundation set, in the order a person reads it. +// +// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live +// under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a +// stream of them is a stream of the least valuable messages the mesh sends, competing for the +// same retention as the ones that matter. +// +// **EVENTS filters on the `event` token**, which is the reason that token exists. A module's +// namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist +// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps +// tools on core NATS, where a lost call is a timeout the caller already handles). +func MeshStreams() []Stream { + return []Stream{ + { + Name: "CONTROL", + // A build's outcome is no longer here: it is the build-machine seat's own event, so one + // publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121). + Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, + Retention: RetentionWorkQueue, + Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " + + "store is away and the message is redelivered; nothing is dropped", + }, + { + Name: "NODES", + Subjects: []string{"mesh.node.*.declare"}, + Retention: RetentionLastPerSubject, + Why: "one declaration per node, always the newest; a node that sees sequence n refuses " + + "n-1 by construction (issue 107)", + }, + { + Name: "EVENTS", + // A seat's own events ride here too: they are 1:many like any event, and the + // `event` token keeps them clear of both the seat's work queue (`accept`) and its + // tools (`tool`), which must not be persisted. + Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}, + Retention: RetentionLimits, + MaxAge: 7 * 24 * 60 * 60, + MaxMsgsPerSubject: 10000, + Why: "a subscriber that was down catches up; tool traffic under the same prefix is " + + "excluded by the event token; per-subject caps keep a noisy emitter from " + + "evicting a quiet one without splitting the stream", + }, + } +} + +// An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it +// keeps this testable without a server, and keeps the client library out of everything that only +// wants to know what the streams are. +type Asserter interface { + // EnsureStream creates the stream if absent and updates it to match if present. It must be + // idempotent: the controller asserts on every start, not only at genesis. + EnsureStream(s Stream) error +} + +// AssertMeshStreams brings the foundation set into being, in order, and says which one failed +// rather than that something did. +// +// Asserted on every start rather than created once at genesis, because a stream that was deleted, +// or a mesh raised from a restored backup, must converge rather than run without the guarantee +// its messages assume. Idempotence is the whole requirement. +func AssertMeshStreams(a Asserter) error { + for _, s := range MeshStreams() { + if err := a.EnsureStream(s); err != nil { + return fmt.Errorf("asserting stream %s: %w", s.Name, err) + } + } + return nil +} + +// Overlaps reports subject filters claimed by more than one stream. +// +// **Corrected against the server**: an earlier version of this comment said NATS accepts +// overlapping streams and stores the message twice. It does not — it refuses the second stream +// with "subjects overlap with an existing stream" (verified against nats-server 2.10). The check +// still earns its place, for a different reason: the server's refusal arrives when the controller +// is applying, naming one stream, at a moment when the mesh is half-configured. This one arrives +// where the set is written, names both, and cannot reach a running mesh. +// +// It also decides a design question. Because overlap is refused rather than merged, a shared +// EVENTS stream and a per-module stream cannot coexist — the module's would be refused — so +// "one stream for most, its own for a module that wants different retention" is not an option +// the server allows. It is all of one or all of the other. +func Overlaps() []string { + seen := map[string]string{} + var clashes []string + for _, s := range MeshStreams() { + for _, subject := range s.Subjects { + if first, ok := seen[subject]; ok { + clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject)) + continue + } + seen[subject] = s.Name + } + } + sort.Strings(clashes) + return clashes +} + +// The mesh's own consumers. +// +// A seat's streams and a module's consumers are derived from declarations (derived.go). These two +// are not: **the controller is not a module and files no manifest**, so its authority and its +// subscriptions cannot come from a declaration that does not exist. They are named here, where the +// mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would +// hear every event in the mesh, which it has no business doing and which would make its permission +// list stop explaining anything. + +// ControllerName is the controller's durable consumer on each stream it reads, and the name its +// ack subject is derived from (nats.go: `$JS.ACK..controller.>`). +const ControllerName = "controller" + +// ControllerFollows are the events the controller reacts to: the catalogue saying a module's +// current version moved, and a catalogue that has just started saying it may have missed builds. +// +// **Derived the same way a module's subscription is**, from the emitter and the bare local event +// name, rather than written out. They were written out while the catalogue still spelled its events +// as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a +// hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from +// the other side. Deriving them means the conversion could not leave these behind. +var ControllerFollows = []string{ + moduleEventSubject("mesh-catalog", "upgraded"), + moduleEventSubject("mesh-catalog", "catching-up"), + // A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a + // message on the control branch. Same three audiences, one publish: whoever asked, this, and the + // catalogue. + seatEventSubject("mesh-build-machine", "built"), +} + +// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so +// what the controller subscribes and what the emitter is permitted to publish cannot drift apart. +func moduleEventSubject(module, event string) string { + return "mesh.mod." + module + ".event." + event +} + +// seatEventSubject is where a role's own event lands, derived the same way a holder's permission is. +func seatEventSubject(seat, verb string) string { + return "mesh.seat." + seat + ".event." + verb +} + +// MeshConsumers is what the controller consumes, in the order a person reads it. +// +// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's, +// not the server's (window.go): a message is held with a nak-and-delay until the controller either +// takes it or gives up and says so. A max-deliver here would dead-letter a push that was being +// held through a store restart — the exact message the stream exists to protect — some minutes +// before the controller had finished deciding about it. +func MeshConsumers() []Consumer { + return []Consumer{ + { + Name: ControllerName, + Stream: "CONTROL", + Push: true, + AckWaitSeconds: 30, + Why: "the controller is the single consumer of what nodes say; explicit ack and no " + + "max-deliver, because the store window's bound is the controller's own", + }, + { + Name: ControllerName, + Stream: "EVENTS", + Filters: ControllerFollows, + Push: true, + AckWaitSeconds: 30, + MaxDeliver: 5, + Why: "the two events the mesh's own controller reacts to; after max-deliver it " + + "dead-letters, because an announcement it cannot act on will not become actionable", + }, + } +} + +// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason +// Asserter is. +type Ensurer interface { + EnsureConsumer(c Consumer) error +} + +// AssertMeshConsumers brings the controller's own consumers into being, and says which one failed. +// +// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the +// refusal names the stream rather than the order. +func AssertMeshConsumers(e Ensurer) error { + for _, c := range MeshConsumers() { + if err := e.EnsureConsumer(c); err != nil { + return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err) + } + } + return nil +} diff --git a/internal/broker/streams_test.go b/internal/broker/streams_test.go new file mode 100644 index 0000000..ca450a3 --- /dev/null +++ b/internal/broker/streams_test.go @@ -0,0 +1,235 @@ +package broker + +import ( + "errors" + "strings" + "testing" +) + +type recorder struct { + seen []Stream + fail string +} + +func (r *recorder) EnsureStream(s Stream) error { + if s.Name == r.fail { + return errors.New("refused") + } + r.seen = append(r.seen, s) + return nil +} + +// The controller asserts on every start, not only at genesis: a stream that was deleted, or a mesh +// raised from a backup, must converge rather than run without the guarantee its messages assume. +func TestAssertingTwiceIsTheSameAsOnce(t *testing.T) { + a, b := &recorder{}, &recorder{} + if err := AssertMeshStreams(a); err != nil { + t.Fatal(err) + } + if err := AssertMeshStreams(a); err != nil { + t.Fatal(err) + } + if err := AssertMeshStreams(b); err != nil { + t.Fatal(err) + } + if len(a.seen) != 2*len(b.seen) { + t.Fatalf("asserted %d then %d; assertion is not repeatable", len(a.seen), len(b.seen)) + } +} + +func TestAFailedAssertionNamesItsStream(t *testing.T) { + err := AssertMeshStreams(&recorder{fail: "NODES"}) + if err == nil || !strings.Contains(err.Error(), "NODES") { + t.Fatalf("got %v, which does not say which stream failed", err) + } +} + +// Two streams matching one subject is accepted by NATS and stores the message twice under two +// retentions. Nothing reports that, so it is refused where the set is written. +func TestNoTwoStreamsClaimTheSameSubject(t *testing.T) { + if clashes := Overlaps(); len(clashes) != 0 { + t.Fatalf("overlapping subject filters: %v", clashes) + } +} + +// A heartbeat under mesh.control.> must not be persisted: a lost one is the next one, and a +// stream of them competes for retention with the messages that matter. +func TestHeartbeatsAreNotInTheControlStream(t *testing.T) { + for _, s := range MeshStreams() { + for _, subject := range s.Subjects { + if subject == "mesh.control.>" || strings.Contains(subject, "alive") { + t.Fatalf("stream %s claims %q, which captures heartbeats", s.Name, subject) + } + } + } +} + +// The reason the kind token exists: a filter over a module's whole namespace would persist every +// tool call in the mesh. +func TestTheEventsStreamDoesNotCaptureToolCalls(t *testing.T) { + var events Stream + for _, s := range MeshStreams() { + if s.Name == "EVENTS" { + events = s + } + } + // Nothing a tool call rides may match any of the filters — a module's or a seat's. + for _, tool := range []string{ + "mesh.mod.billing.tool.status", + "mesh.seat.telegram-sender.tool.status", + "mesh.seat.telegram-sender.accept.send", // work, not an event: its own stream + } { + for _, f := range events.Subjects { + if subjectMatches(f, tool) { + t.Fatalf("%q matches the events filter %q, so it would be persisted here", tool, f) + } + } + } + // And both kinds of event do match. + for _, event := range []string{ + "mesh.mod.billing.event.order.placed", + "mesh.seat.telegram-sender.event.delivered", + } { + matched := false + for _, f := range events.Subjects { + if subjectMatches(f, event) { + matched = true + } + } + if !matched { + t.Fatalf("%q matches no events filter, so nothing would keep it", event) + } + } +} + +// subjectMatches is NATS subject matching, enough for these filters: `*` is one token, `>` is the +// rest. +func subjectMatches(filter, subject string) bool { + f, s := strings.Split(filter, "."), strings.Split(subject, ".") + for i, tok := range f { + if tok == ">" { + return i <= len(s) + } + if i >= len(s) { + return false + } + if tok != "*" && tok != s[i] { + return false + } + } + return len(f) == len(s) +} + +// Each relationship's retention is the thing that makes it what it is (design 29 §4). +func TestEachStreamCarriesTheRetentionItsShapeNeeds(t *testing.T) { + want := map[string]Retention{ + "CONTROL": RetentionWorkQueue, + "NODES": RetentionLastPerSubject, + "EVENTS": RetentionLimits, + } + got := map[string]Retention{} + for _, s := range MeshStreams() { + got[s.Name] = s.Retention + if s.Why == "" { + t.Errorf("stream %s says no reason it exists", s.Name) + } + } + if len(got) != len(want) { + t.Fatalf("the foundation set is %v", got) + } + for name, r := range want { + if got[name] != r { + t.Errorf("%s retains as %q, expected %q", name, got[name], r) + } + } +} + +// The order the bus's objects are asserted in, because getting it wrong is a refusal that names the +// wrong thing: a consumer on a stream that does not exist is refused naming the *stream*, so +// somebody reading it goes looking for a deletion instead of a reversed pair of lines. +func TestTheBusesObjectsAreAssertedStreamsBeforeConsumers(t *testing.T) { + r := &recording{} + if err := Raise(r, []string{"anchor", "laptop"}); err != nil { + t.Fatal(err) + } + + // Every stream before every consumer. + firstConsumer := -1 + for i, step := range r.steps { + if strings.HasPrefix(step, "consumer ") && firstConsumer < 0 { + firstConsumer = i + } + if strings.HasPrefix(step, "stream ") && firstConsumer >= 0 { + t.Fatalf("a stream was asserted after a consumer: %v", r.steps) + } + } + if firstConsumer < 0 { + t.Fatalf("no consumer was asserted: %v", r.steps) + } + + // And every node got one, named after it — without which that node hears nothing while + // everything else about it looks correct. + for _, node := range []string{"anchor", "laptop"} { + if !containsStep(r.steps, "consumer NODES/"+node) { + t.Errorf("%s was given no way to hear its declaration: %v", node, r.steps) + } + } + // And the controller its own, on both streams it reads. + for _, want := range []string{"consumer CONTROL/controller", "consumer EVENTS/controller"} { + if !containsStep(r.steps, want) { + t.Errorf("the controller is missing %s: %v", want, r.steps) + } + } +} + +// A seat's work queue is asserted whether or not anybody holds it; the holder's worker only when +// somebody does. **The stream without the consumer is the point**: work queues until a holder +// appears, so installing the module later flushes the backlog instead of having lost it. +func TestASeatsQueueExistsBeforeItsHolderDoes(t *testing.T) { + seats := []DeclaredSeat{{Name: "telegram-sender", Accepts: []string{"send"}}} + + unheld := &recording{} + if err := RaiseSeats(unheld, seats, nil); err != nil { + t.Fatal(err) + } + if !containsStep(unheld.steps, "stream SEAT_TELEGRAM_SENDER") { + t.Fatalf("a declared seat got no work queue: %v", unheld.steps) + } + for _, step := range unheld.steps { + if strings.HasPrefix(step, "consumer ") { + t.Fatalf("a seat nobody holds got a worker: %v", unheld.steps) + } + } + + held := &recording{} + if err := RaiseSeats(held, seats, map[string]Holder{ + "telegram-sender": {Node: "anchor", Module: "telegram"}, + }); err != nil { + t.Fatal(err) + } + if !containsStep(held.steps, "consumer SEAT_TELEGRAM_SENDER/SEAT_TELEGRAM_SENDER_worker") { + t.Fatalf("the seat's holder got no worker: %v", held.steps) + } +} + +// recording is a connection to the bus that writes down what it was asked for. +type recording struct{ steps []string } + +func (r *recording) EnsureStream(s Stream) error { + r.steps = append(r.steps, "stream "+s.Name) + return nil +} + +func (r *recording) EnsureConsumer(c Consumer) error { + r.steps = append(r.steps, "consumer "+c.Stream+"/"+c.Name) + return nil +} + +func containsStep(steps []string, want string) bool { + for _, s := range steps { + if s == want { + return true + } + } + return false +} diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf new file mode 100644 index 0000000..f64b2e8 --- /dev/null +++ b/internal/broker/testdata/composed.conf @@ -0,0 +1,53 @@ +# Composed by the mesh controller. Do not edit: the next composition overwrites it. +# Accounts and permissions are derived from what each module declares and nothing +# else (novox/hq ADR 0043, design 29 §2). + +port: 4222 +http: 127.0.0.1:8222 + +tls { + cert_file: "/tls/tls.crt" + key_file: "/tls/tls.key" + ca_file: "/tls/ca.crt" +} + +jetstream { + store_dir: "/data" +} + +# The mesh's users, composed by the controller. Do not edit: the next +# composition overwrites it. Permissions are derived from what each module +# declares and nothing else (novox/hq ADR 0043, design 29 §2). + +accounts { + MESH { + users = [ + { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.control.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>"] } + subscribe: { allow: ["$JS.API.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built"] } + allow_responses: { max: 1, ttl: "1m" } + } } + { user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: { + publish: { allow: ["mesh.control.enrol"] } + subscribe: { allow: ["_INBOX.enrol.one.>"] } + } } + { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: { + publish: { allow: ["$JS.ACK.NODES.one.>", "mesh.control.one.>"] } + subscribe: { allow: ["_INBOX.node.one.>", "mesh.node.one.declare"] } + } } + { user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] } + subscribe: { allow: ["_INBOX.one.telegram.>", "mesh.mod.telegram.tool.status", "mesh.seat.telegram-sender.accept.send"] } + allow_responses: { max: 1, ttl: "1m" } + } } + { user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.two_audit.>"] } + subscribe: { allow: ["_INBOX.two.audit.>", "mesh.mod.shop.event.order.placed"] } + } } + { user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: { + publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] } + subscribe: { allow: ["_INBOX.two.shop.>"] } + } } + ] + } +} diff --git a/internal/broker/users.go b/internal/broker/users.go new file mode 100644 index 0000000..17d0ce4 --- /dev/null +++ b/internal/broker/users.go @@ -0,0 +1,127 @@ +package broker + +import ( + "fmt" + "sort" +) + +// Every user the composed file should contain, derived from what the mesh knows. +// +// **The list is derived, never kept.** A stored user list would be a second account of who may +// reach the bus, able to disagree with the records it came from — and the disagreement would be +// invisible, because both would look internally consistent. So this is a pure function of the +// mesh's records, run again every time the file is written. +// +// Records are mirrored into this package's own types rather than imported from the catalogue, for +// the reason DeclaredSeat is: composing authority is a different job from parsing a manifest, and +// this package stays free of the other's types so a change to a manifest field cannot quietly widen +// a permission. + +// Declared is one module on one node, as composing its authority needs it. +type Declared struct { + Module string + Emits []string + Consumes []string + Serves []string + // Holds are the seats this module claims, with the protocol each seat declares. A seat the + // mesh defines for itself declares no protocol, so holding one grants nothing on the bus — + // which is right: those seats are about who does a job, not about who may say what. + Holds []Seat + // Uses are the seats this module sends to. + Uses []Seat + // Watches are the seats whose events it consumes. + Watches []Seat +} + +// Records is what composing a user list needs to know about the mesh, and nothing more. +type Records struct { + // Nodes is every machine the mesh knows. Each gets a host user. + Nodes []string + // Assigned is the modules on each node, as they declare themselves. + Assigned map[string][]Declared + // Enrolling is every node with a live token — one enrolment user each, because the inbox an + // answer goes to is scoped to the token and a shared one is one machine reading another's + // sealed credentials (design 25 §6). + Enrolling []string + // People is each person's name against the tools they may invoke, `*` for an administrator. + People map[string][]string +} + +// Users is every user the composed file should contain, in the order it will be written. +// +// The controller is always first and always present: a mesh whose own controller is not in the file +// is a mesh that cannot be told anything, and there is no state of the records in which that is +// correct. +func Users(r Records) ([]Principal, error) { + out := []Principal{{Kind: KindController}} + + for _, node := range sortedCopy(r.Nodes) { + out = append(out, Principal{Kind: KindNode, Node: node}) + for _, d := range r.Assigned[node] { + out = append(out, Principal{ + Kind: KindModule, Node: node, Module: d.Module, + Emits: d.Emits, Consumes: d.Consumes, Serves: d.Serves, + Holds: d.Holds, Uses: d.Uses, Watches: d.Watches, + }) + } + } + for _, node := range sortedCopy(r.Enrolling) { + out = append(out, Principal{Kind: KindEnrolment, Node: node}) + } + for _, person := range sortedNames(r.People) { + out = append(out, Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]}) + } + + // Refused here rather than discovered by the server. Two users with one name is a file the + // server reads as one of them, and which one depends on the order — so a module assigned to a + // node twice, or a person named after nothing, is a composition that must not be written. + seen := map[string]string{} + for _, p := range out { + name := p.Username() + if name == "" || name == "." { + return nil, fmt.Errorf("a %s user has no name, so nothing could authenticate as it", p.Kind) + } + if first, already := seen[name]; already { + return nil, fmt.Errorf( + "two users would be called %q (a %s and a %s): the server would read the file as "+ + "one of them, and which one depends on the order", name, first, p.Kind) + } + seen[name] = string(p.Kind) + } + return out, nil +} + +// WithPasswords fills each user's hash from what the mesh minted, and says which users have none. +// +// **Separated from Users because they fail differently.** A user missing from the records is a bug +// in deriving them; a user with no password is a step that has not happened yet — a module assigned +// but never given a credential, a node enrolled before this existed. The second is ordinary and its +// remedy is to mint one, so it is named rather than returned as an error, and the caller decides +// whether a partial composition is worth writing. +func WithPasswords(principals []Principal, hashes map[string]string) (filled []Principal, missing []string) { + for _, p := range principals { + hash, ok := hashes[p.Username()] + if !ok || hash == "" { + missing = append(missing, p.Username()) + continue + } + p.PasswordHash = hash + filled = append(filled, p) + } + return filled, missing +} + +func sortedCopy(in []string) []string { + out := append([]string(nil), in...) + sort.Strings(out) + return out +} + +func sortedNames(in map[string][]string) []string { + out := make([]string, 0, len(in)) + for k := range in { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/broker/users_test.go b/internal/broker/users_test.go new file mode 100644 index 0000000..0e21eaf --- /dev/null +++ b/internal/broker/users_test.go @@ -0,0 +1,241 @@ +package broker + +import ( + "strings" + "testing" +) + +// Deriving the bus's user list from the mesh's records. +// +// Every test here is about a way the list could be wrong that the server would not tell anybody +// about: a user missing, a user named twice, a user with authority it did not declare. + +func someRecords() Records { + return Records{ + Nodes: []string{"two", "one"}, + Assigned: map[string][]Declared{ + "one": {{Module: "telegram", Serves: []string{"status"}}}, + "two": {{Module: "shop", Emits: []string{"order.placed"}}}, + }, + Enrolling: []string{"three"}, + People: map[string][]string{"ada": {"mesh-catalog.catalog_tools"}}, + } +} + +func namesOf(t *testing.T, r Records) []string { + t.Helper() + users, err := Users(r) + if err != nil { + t.Fatal(err) + } + out := make([]string, 0, len(users)) + for _, u := range users { + out = append(out, u.Username()) + } + return out +} + +// The controller is always there. A mesh whose own controller is not in the file is a mesh that +// cannot be told anything, and there is no state of the records in which that is correct. +func TestTheControllerIsAlwaysInTheList(t *testing.T) { + for _, r := range []Records{{}, someRecords()} { + names := namesOf(t, r) + if len(names) == 0 || names[0] != "controller" { + t.Fatalf("the controller is not first in %v", names) + } + } +} + +// One user per node, one per module per node, one per live token and one per person — and nothing +// else, because a user nobody derived is a user nobody can explain. +func TestEveryRecordBecomesExactlyOneUser(t *testing.T) { + names := namesOf(t, someRecords()) + want := []string{ + "controller", + "node.one", "one.telegram", + "node.two", "two.shop", + "enrol.three", + "person.ada", + } + if strings.Join(names, ",") != strings.Join(want, ",") { + t.Fatalf("derived %v\n want %v", names, want) + } +} + +// Two users with one name is a file the server reads as one of them, and which one depends on the +// order. Refused here, where both can be named, rather than left to be whichever the server picked. +func TestTwoUsersWithOneNameAreRefused(t *testing.T) { + r := someRecords() + r.Assigned["one"] = append(r.Assigned["one"], Declared{Module: "telegram"}) + _, err := Users(r) + if err == nil { + t.Fatal("a module assigned twice to one node composed two users with one name") + } + if !strings.Contains(err.Error(), "one.telegram") { + t.Fatalf("the refusal does not name the user: %v", err) + } +} + +// A module's authority is what it declared and nothing more, carried through the derivation intact — +// because this is the step where a mistake would grant something no manifest asked for. +func TestAModulesAuthorityIsWhatItDeclared(t *testing.T) { + seat := Seat{Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}} + users, err := Users(Records{ + Nodes: []string{"one"}, + Assigned: map[string][]Declared{"one": {{ + Module: "shop", Emits: []string{"order.placed"}, Uses: []Seat{seat}, + }}}, + }) + if err != nil { + t.Fatal(err) + } + perms, err := PermissionsFor(users[len(users)-1]) + if err != nil { + t.Fatal(err) + } + has(t, perms.Publish, "mesh.mod.shop.event.order.placed") + has(t, perms.Publish, "mesh.seat.telegram-sender.accept.send") + // A seat it uses, not one it holds: it may submit work and may not publish the seat's own + // events, or it could lie about outcomes on a role somebody else fills. + hasNot(t, perms.Publish, "mesh.seat.telegram-sender.event.delivered") + hasNot(t, perms.Subscribe, "mesh.seat.telegram-sender.accept.send") +} + +// A user the mesh has never minted a password for is named rather than silently dropped or +// composed as a user anybody is. It is an ordinary situation — a module assigned a moment ago — and +// the remedy is to mint one, so the caller decides whether to write a partial file. +func TestAUserWithNoPasswordIsNamedRatherThanWritten(t *testing.T) { + users, err := Users(someRecords()) + if err != nil { + t.Fatal(err) + } + filled, missing := WithPasswords(users, map[string]string{ + "controller": "$2a$hash", "node.one": "$2a$hash", + }) + if len(filled) != 2 { + t.Fatalf("composed %d users from two hashes", len(filled)) + } + if len(missing) != len(users)-2 { + t.Fatalf("%d users are missing a password, of %d: %v", len(missing), len(users), missing) + } + for _, p := range filled { + if p.PasswordHash == "" { + t.Fatalf("%s was kept with no password, which is a user anybody is", p.Username()) + } + } +} + +// And the whole thing composes: records in, a file the server would read out. +func TestRecordsComposeIntoAFile(t *testing.T) { + users, err := Users(someRecords()) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for _, u := range users { + hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22) + } + filled, missing := WithPasswords(users, hashes) + if len(missing) != 0 { + t.Fatalf("users with no password: %v", missing) + } + got, err := Compose(Server{ClientPort: 4222, MonitoringPort: 8222, StoreDir: "/data", + TLSCert: "/tls/tls.crt", TLSKey: "/tls/tls.key", TLSCA: "/tls/ca.crt"}, filled) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `user: "controller"`, `user: "node.one"`, `user: "one.telegram"`, + `user: "enrol.three"`, `user: "person.ada"`, + `"_INBOX.enrol.three.>"`, `"mesh.mod.mesh-catalog.tool.catalog_tools"`, + } { + if !strings.Contains(got, want) { + t.Errorf("the composed file does not contain %s", want) + } + } +} + +// The accounts block alone is what the mesh writes, and it holds nothing about the server. +// +// **The split is the whole design decision** (ComposeAccounts): ports, TLS paths and a store +// directory are properties of the container the module raises, and a controller that wrote them +// would have to be kept in step with a Dockerfile it never sees. So this test says what must not be +// in the file as plainly as what must. +func TestWhatTheMeshWritesIsUsersAndNothingAboutTheServer(t *testing.T) { + users, err := Users(someRecords()) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for _, u := range users { + hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22) + } + filled, missing := WithPasswords(users, hashes) + if len(missing) != 0 { + t.Fatalf("users with no password: %v", missing) + } + got, err := ComposeAccounts(filled) + if err != nil { + t.Fatal(err) + } + + for _, want := range []string{"accounts {", `user: "controller"`, `user: "one.telegram"`} { + if !strings.Contains(got, want) { + t.Errorf("the accounts file does not contain %s", want) + } + } + // None of the server's own settings. Each of these in the mesh's file is a value the controller + // would then own, and the module could no longer change its own image without the mesh agreeing. + for _, absent := range []string{"port:", "http:", "jetstream", "tls {", "store_dir", "cert_file"} { + if strings.Contains(got, absent) { + t.Errorf("the accounts file contains %q, which belongs to the module that raises the "+ + "server, not to the mesh", absent) + } + } +} + +// A user with no password is refused here too, not only by the whole-file composition: this is the +// function the controller actually calls, and a user without a password is a user anybody is. +func TestTheAccountsFileRefusesAUserWithNoPassword(t *testing.T) { + if _, err := ComposeAccounts([]Principal{{Kind: KindController}}); err == nil { + t.Fatal("a user with no password hash was written") + } +} + +// **A user list is composed for a bus the mesh has not moved onto yet**, and that is the whole of +// step 2 (novox/hq ADR 0116): the server stands in the mesh carrying nothing, on its own ports, while +// every node is still on the bus it was on. +// +// Pinned because the first version of the composing step got it backwards — it wrote the list only +// once the controller was already on the new bus, which is a step that cannot be taken: the module +// comes up, finds no accounts file, and waits for one the controller had decided not to write. +func TestAUserListIsComposedBeforeAnythingMovesOntoTheBus(t *testing.T) { + // Exactly the records of a mesh mid-change: everything running, nothing on the new bus. + users, err := Users(Records{ + Nodes: []string{"anchor"}, + Assigned: map[string][]Declared{"anchor": {{Module: "nats"}}}, + }) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for _, u := range users { + hashes[u.Username()] = "$2a$11$" + strings.Repeat("x", 22) + } + filled, missing := WithPasswords(users, hashes) + if len(missing) != 0 { + t.Fatalf("users with no credential: %v", missing) + } + accounts, err := ComposeAccounts(filled) + if err != nil { + t.Fatal(err) + } + // The controller's own user above all: a file without it is a bus its writer cannot connect to, + // which is what the server would be left holding the moment it starts. + if !strings.Contains(accounts, `user: "controller"`) { + t.Fatalf("the composed list does not contain the controller:\n%s", accounts) + } + if !strings.Contains(accounts, `user: "node.anchor"`) { + t.Errorf("the composed list does not contain the machine running the bus") + } +} diff --git a/internal/catalogue/broker_seat_test.go b/internal/catalogue/broker_seat_test.go new file mode 100644 index 0000000..20c33c5 --- /dev/null +++ b/internal/catalogue/broker_seat_test.go @@ -0,0 +1,101 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// The mesh's bus is one per mesh, read from the catalogue beside this checkout. +// +// **This is step 2's claim, and it is checked here rather than in a bed** (novox/hq ADR 0116): +// adoption puts the NATS server into the `mesh-broker` seat on a mesh that is already running, +// and the property that matters is that a second one anywhere is refused *when it is assigned*, +// not discovered later as two servers holding different halves of the mesh's traffic. A second +// bus is not a degraded mesh; it is two meshes that both believe they are the one. +func TestASecondMeshBusAnywhereIsRefusedByName(t *testing.T) { + nats := catalogueManifest(t, "nats") + + if _, err := Resolve(shelf(nats), []string{"nats"}, workstation(), World{}); err != nil { + t.Fatalf("the bus alone does not resolve: %v", err) + } + + elsewhere := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh, + Node: "anchor", Module: "nats"}}} + other := workstation() + other.Name = "laptop" + _, err := Resolve(shelf(nats), []string{"nats"}, other, elsewhere) + if err == nil { + t.Fatal("a second bus was accepted on another machine") + } + if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "one per mesh") { + t.Fatalf("refused without naming the seat: %v", err) + } +} + +// The seat is the server's role, not the product's name (novox/hq ADR 0079). A different +// implementation of the bus claims the same seat, and the mesh refuses it for the same reason — +// which is the property that lets the bus be replaced at all. +func TestTheSeatRefusesADifferentBusToo(t *testing.T) { + nats := catalogueManifest(t, "nats") + held := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh, + Node: "anchor", Module: "some-other-broker"}}} + other := workstation() + other.Name = "laptop" + if _, err := Resolve(shelf(nats), []string{"nats"}, other, held); err == nil { + t.Fatal("the seat admitted a second holder because the module's name differed") + } +} + +// The old broker no longer claims the seat: it is an ordinary provider of `amqp` +// (novox/hq ADR 0119), so it can sit on the same mesh as the bus without contending for it. +func TestTheAmqpBrokerDoesNotContendForTheSeat(t *testing.T) { + lavinmq := catalogueManifest(t, "lavinmq") + for _, c := range lavinmq.Claims { + if c.Name == "mesh-broker" { + t.Fatal("the amqp broker still claims mesh-broker; it is a provider, not foundation") + } + } + busHeld := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh, + Node: "anchor", Module: "nats"}}} + other := workstation() + other.Name = "laptop" + if _, err := Resolve(shelf(lavinmq), []string{"lavinmq"}, other, busHeld); err != nil { + t.Fatalf("the amqp broker was refused beside the mesh bus: %v", err) + } +} + +// **A seat and the interface it delivers are different names, and renaming one must not rename +// the other** (novox/hq ADR 0118). This nearly went wrong: the seats were renamed to the `mesh-*` +// prefix, and a blanket search-and-replace also renamed `npm-package-registry` and `git` where +// they are *provisions* — which a consumer requires and a provider offers. The tests failed with +// "the package registry is served on ", which does not say "you renamed an interface". +func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) { + for _, pair := range []struct{ seat, delivers string }{ + {"git", "git"}, + {"npm-package-registry", "npm-package-registry"}, + {"the-artifact-store", "artifact-store"}, + {"mesh-store", "postgres-database"}, + {"mesh-broker", "mesh-bus"}, + } { + s, known := SeatNamed(pair.seat) + if !known { + t.Fatalf("%q is not a seat", pair.seat) + } + if s.Delivers != pair.delivers { + t.Errorf("the %s seat delivers %q, expected %q — renaming the seat moved the "+ + "interface with it, and every consumer requiring it would stop resolving", + pair.seat, s.Delivers, pair.delivers) + } + // **Three of these deliberately share a name with what they deliver**, and that is not an + // incomplete rename. Renaming a seat that delivers a provision cascades to every consumer + // requiring it, with a mesh-wide window where a holder stops resolving mid-flight — so the + // trunk deferred exactly those three (novox/hq ADR 0121) while renaming the node-scoped ones. + // What this test is for is the other direction: that renaming a seat never moves the + // interface, which once produced "the package registry is served on ". + } +} + +// A manifest written against an old seat name is told what it became rather than refused as +// unknown. **That map is the controller's store now, not this package** (novox/hq ADR 0122): a +// rename is a row, so the courtesy survives a rename nobody recompiled for. Checked where the +// table is read, not here, where there is no longer a hardcoded list to check against. diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 5cd726e..1a9a410 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -97,6 +97,13 @@ type Rendering struct { // compose it a second time. Suffix string + // BusUsers is the mesh's composed user list, for the module holding `mesh-broker`. Empty on + // every other node, and on this one until the controller has composed it. + // + // **Only the users, never the server's own settings**: those are the module's, in its image and + // its mounts (Manifest.BusUsers). + BusUsers string + // MeshRange is the private network's CIDR (the range node addresses are allocated from), for a // module that must name the whole mesh rather than one machine — an intrusion filter that must // never ban a tunnel peer, say. A per-mesh value the module cannot know, so it is carried here @@ -376,6 +383,35 @@ func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[stri }) } } + if m.BusUsers != "" { + // **The claim authorises it, not the field.** This file holds every user's password + // hash, so a module that could ask for it could read every credential on the bus. + // Checked from this manifest alone, which is the cheapest check there is: whether some + // other module also claims the seat is resolution's business elsewhere, and one holder + // mesh-wide is already guaranteed. + if !m.ClaimsSeat("mesh-broker") { + return nil, fmt.Errorf( + "%s asks for the mesh's user list and does not claim mesh-broker. That file "+ + "holds every user's password hash, so the seat is what authorises it", + m.Module) + } + if with.BusUsers == "" { + // Asked for and not composed. Refused rather than skipped, for the reason a + // certificate is: a bus with no user list refuses every connection in the mesh, and + // an empty file would look like a configuration problem on the machine. + return nil, fmt.Errorf( + "%s holds mesh-broker and the mesh composed no user list, so the bus would "+ + "refuse every connection", m.Module) + } + first = append(first, map[string]any{ + "id": BusUsersID(), "type": "file", "path": m.BusUsers, + "content": with.BusUsers, + // Readable by the server and nothing else. Hashes rather than passwords, so this is + // not a set of working credentials — but a list of every user in the mesh is worth + // keeping to the one process that needs it. + "mode": "0600", + }) + } for _, name := range sortedKeys(m.OwnSecrets) { sealed := with.Needed[m.Module][name] if sealed == "" { diff --git a/internal/catalogue/declaration_test.go b/internal/catalogue/declaration_test.go new file mode 100644 index 0000000..23b1caa --- /dev/null +++ b/internal/catalogue/declaration_test.go @@ -0,0 +1,80 @@ +package catalogue + +import "testing" + +// The mesh's user list reaches the module holding the bus, and nothing else. +// +// Three refusals and one delivery, because each of the refusals would be silent in a different way: +// a module that asked and was given it could read every credential on the bus; a bus given an empty +// file refuses every connection in the mesh and looks like a machine problem; and a bus that never +// asked gets nothing rather than a file it does not read. +func TestTheMeshsUserListGoesOnlyToTheModuleHoldingTheBus(t *testing.T) { + theBus := func() Manifest { + return Manifest{ + Module: "nats", Version: "1", + Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}, + BusUsers: "/var/lib/nats-module/conf/accounts.conf", + Resources: []map[string]any{}, + } + } + + on := func(t *testing.T, m Manifest, with Rendering) ([]map[string]any, error) { + t.Helper() + return Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(with) + } + + t.Run("the holder is given it", func(t *testing.T) { + resources, err := on(t, theBus(), Rendering{BusUsers: "accounts { MESH { users = [] } }"}) + if err != nil { + t.Fatal(err) + } + // Prefixed with the module it came from, like every resource: two modules may reasonably + // both call something "config", and without the prefix the second would silently replace + // the first. + var found map[string]any + for _, r := range resources { + if r["id"] == "nats."+BusUsersID() { + found = r + } + } + if found == nil { + t.Fatalf("the bus was given no user list: %+v", resources) + } + if found["path"] != "/var/lib/nats-module/conf/accounts.conf" { + t.Errorf("written to %v rather than where the module asked", found["path"]) + } + if found["mode"] != "0600" { + t.Errorf("mode %v: a list of every user in the mesh belongs to the one process that "+ + "needs it", found["mode"]) + } + }) + + t.Run("a module that does not claim the seat is refused", func(t *testing.T) { + m := theBus() + m.Claims = nil + if _, err := on(t, m, Rendering{BusUsers: "accounts {}"}); err == nil { + t.Fatal("a module that claims nothing was handed every user's password hash") + } + }) + + t.Run("the holder with nothing composed is refused", func(t *testing.T) { + if _, err := on(t, theBus(), Rendering{}); err == nil { + t.Fatal("the bus was given an empty user list, so it would refuse every connection in " + + "the mesh and look like a machine problem") + } + }) + + t.Run("a module that did not ask gets nothing", func(t *testing.T) { + m := theBus() + m.BusUsers = "" + resources, err := on(t, m, Rendering{BusUsers: "accounts {}"}) + if err != nil { + t.Fatal(err) + } + for _, r := range resources { + if r["id"] == "nats."+BusUsersID() { + t.Fatal("a module that asked for no user list was given one") + } + } + }) +} diff --git a/internal/catalogue/events.go b/internal/catalogue/events.go new file mode 100644 index 0000000..90dda63 --- /dev/null +++ b/internal/catalogue/events.go @@ -0,0 +1,161 @@ +package catalogue + +import ( + "fmt" + "regexp" + "strings" +) + +// What a module may call an event, and what a consumer may ask for. +// +// A module names an event **locally**: `order.placed`, not a subject and not a routing key +// (design 29 §1). A consumer names the emitter and the event: `billing.order.placed`. The mesh +// derives the subject from those, so reorganising the subject space leaves every manifest correct. +// +// **Nothing checked this until every manifest in the catalogue was wrong the same way** +// (novox/hq 04-ISSUES/127). All thirty-seven kept the old bus's routing key — +// `module..` — which the derivation read as "a module called `module`", so every +// cross-module subscription in the mesh pointed at a namespace nobody publishes to. Nothing failed: +// the services started and none of them reacted. The documentation on these fields taught the old +// form too, which is why the drift was uniform rather than scattered. + +// eventName is one name in a local event: lower-case, and no wildcard. +var eventName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// The wildcards a consumer may use, spelled the mesh's way and derived to whatever the transport +// spells them as. +// +// **A manifest holds no transport token**, which is the whole point of naming locally: the bus the +// mesh runs on today spells these `*` and `#`, and the one being built spells them `*` and `>`. A +// manifest that said either would be a manifest that stopped being true when the wire changed. +const ( + // OneName stands for exactly one name. + OneName = "*" + // TheRest stands for one or more names, and may only come last. + TheRest = "**" +) + +// EventProblems is what is wrong with a manifest's events. +// +// Refused at registration, because the alternative is a module that installs, starts, connects and +// reacts to nothing — and every log line says it is fine. +func EventProblems(m Manifest) []string { + var problems []string + + for _, e := range m.Emits { + if was, stale := staleEventForm(e, m.Module); stale { + problems = append(problems, fmt.Sprintf( + "%s emits %q, which is the old bus's routing key. An event is named locally now, so "+ + "write %q — the mesh derives the subject (novox/hq design 29 §1)", + m.Module, was, strings.TrimPrefix(was, "module."+m.Module+"."))) + continue + } + if strings.HasPrefix(e, "module.") { + problems = append(problems, fmt.Sprintf( + "%s emits %q: `module.` is reserved, because it is how the old bus spelled a "+ + "routing key and an event named that way derives into a namespace nobody owns", + m.Module, e)) + continue + } + if err := localName(e); err != nil { + problems = append(problems, fmt.Sprintf("%s emits %q: %v", m.Module, e, err)) + continue + } + // **Its own name, never another's.** The bus enforces that a namespace belongs to the module + // it is named for, so an event named for somebody else cannot be published at all. If the + // event is about a role rather than about this module, it belongs on the seat: a name that + // is stable across whoever fills it (04-ISSUES/127). + if first, _, split := strings.Cut(e, "."); split && isAModuleNameOtherThan(first, m.Module) { + problems = append(problems, fmt.Sprintf( + "%s emits %q, which reads as another module's event. A module publishes under its "+ + "own name only. If this is about a role rather than about %s, declare it on that "+ + "seat, where the name survives the holder changing", + m.Module, e, m.Module)) + } + } + + for _, c := range m.Consumes { + if strings.HasPrefix(c, "module.") { + problems = append(problems, fmt.Sprintf( + "%s consumes %q, which is the old bus's pattern. A consumed event names its emitter "+ + "and the event: write %q", m.Module, c, strings.TrimPrefix(c, "module."))) + continue + } + if c == "#" { + problems = append(problems, fmt.Sprintf( + "%s consumes %q, which is the old bus's wildcard for everything. Write %q", + m.Module, c, TheRest)) + continue + } + if err := consumePattern(c); err != nil { + problems = append(problems, fmt.Sprintf("%s consumes %q: %v", m.Module, c, err)) + } + } + return problems +} + +// staleEventForm says an emitted name is this module's own old routing key, and what it was. +func staleEventForm(event, module string) (string, bool) { + return event, module != "" && strings.HasPrefix(event, "module."+module+".") +} + +// isAModuleNameOtherThan says a first token names some module of this mesh that is not this one. +// +// Only the mesh's own seats and the catalogue could answer this properly, and neither is reachable +// from a parser given one manifest. So this catches the case that actually happened — a name that +// is a *provision* the mesh defines, which is where "another module's event" comes from in practice +// — and the whole-catalogue check catches the rest. +func isAModuleNameOtherThan(first, module string) bool { + if first == module || first == "" { + return false + } + if _, isASeat := SeatNamed(first); isASeat { + return true + } + if _, isASeat := SeatDelivering(first); isASeat { + return true + } + return false +} + +// localName checks one event name: dot-separated names, no wildcards, nothing else. +func localName(event string) error { + if event == "" { + return fmt.Errorf("an event needs a name") + } + for _, part := range strings.Split(event, ".") { + if part == OneName || part == TheRest { + return fmt.Errorf("an emitted event names one event, so it carries no wildcard") + } + if !eventName.MatchString(part) { + return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part) + } + } + return nil +} + +// consumePattern checks a consumed pattern: the emitter, then the event, with wildcards. +func consumePattern(pattern string) error { + if pattern == "" { + return fmt.Errorf("a consumed event needs an emitter and an event") + } + parts := strings.Split(pattern, ".") + for i, part := range parts { + switch { + case part == TheRest: + if i != len(parts)-1 { + return fmt.Errorf("%q stands for the rest of a name, so nothing may follow it", TheRest) + } + case part == OneName: + case !eventName.MatchString(part): + return fmt.Errorf("%q is not a usable name: lower-case letters, digits and dashes", part) + } + } + // `**` alone is every event from every module, which the audit logger wants and says plainly. + if len(parts) == 1 && parts[0] != TheRest { + return fmt.Errorf( + "%q names an emitter and no event. Write ., or %q for every event", + pattern, TheRest) + } + return nil +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index fca2fa8..64a67bc 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -176,15 +176,29 @@ type Manifest struct { // Requires are names that must be provided by something assigned to the same node. Requires []string `json:"requires,omitempty"` - // Emits are the event types this module publishes onto the broker — dotted topic keys, e.g. - // "module.umami.site.created". Declared so the mesh knows the event graph; events are - // provisioning's lighter sibling — 1:many and broadcast, no credential (novox/hq ADR 0041). + // Emits are the events this module publishes, named **locally**: `order.placed`, not a subject + // and not a routing key. The mesh derives where it lands (design 29 §1), so reorganising the + // subject space leaves this manifest correct. Events are provisioning's lighter sibling — 1:many + // and broadcast, no credential (novox/hq ADR 0041). + // + // A module publishes under its own name only. If the event is about a *role* rather than about + // this module, it belongs on that seat, where the name outlives whoever holds it. + // + // **This said "dotted topic keys, e.g. module.umami.site.created" until 04-ISSUES/127**, which + // is the old bus's routing key, and is why every manifest in the catalogue had the same mistake: + // nobody was guessing, everybody followed this comment. Emits []string `json:"emits,omitempty"` - // Consumes are the event patterns this module subscribes to — topic patterns over module, - // mesh and node events alike, e.g. "node.*.joined" or "#" (the audit logger). The runtime - // wires the subscription; the module ships the handler. A Consumes for an event nothing on - // the mesh Emits is a dangling edge. + // Consumes are the events this module reacts to, each naming its emitter and the event: + // `billing.order.placed`. `*` stands for one name and `**` for the rest, so `*.download.completed` + // is that event from any module and `**` is every event in the mesh. + // + // Spelled the mesh's way rather than the wire's, for the reason Emits is: the bus the mesh runs + // on today spells these `*` and `#`, the one being built spells them `*` and `>`, and a manifest + // naming either would stop being true when the wire changed. + // + // The runtime wires the subscription; the module ships the handler. A Consumes for an event + // nothing on the mesh Emits is a dangling edge. Consumes []string `json:"consumes,omitempty"` // Claims are singular resources. Two modules claiming one thing within a scope cannot both @@ -192,13 +206,33 @@ type Manifest struct { // that every new module would force its predecessors to update. Claims []Claim `json:"claims,omitempty"` - // DefinesSeats are the seats this module defines for itself (novox/hq ADR 0121). The control - // plane defines the system seats — `mesh-*` and `node-*` — and a module may define its own, - // named outside that namespace, to coordinate its own instances: the mesh enforces - // one-holder-per-scope for it without knowing what it means. A module's declared seat is the - // only non-system name it may then claim; a claim to a name neither the mesh nor the module - // defines is refused. - DefinesSeats []Claim `json:"seats,omitempty"` + // DefinesSeats are the seats this module defines for itself, with their protocols + // (novox/hq ADR 0121, ADR 0129). The control plane defines the system seats — `mesh-*` and + // `node-*` — and a module may define its own, named outside that namespace, to coordinate its + // own instances: the mesh enforces one-holder-per-scope for it without knowing what it means. A + // module's declared seat is the only non-system name it may then claim; a claim to a name + // neither the mesh nor the module defines is refused. + // + // **Two lines of work built this at once**, one calling it `Seats` with a protocol and one + // `DefinesSeats` without. Same key in the file, so no manifest is affected: this is the trunk's + // name with the richer type, because what a role accepts, emits and serves is what lets the mesh + // check that a holder answers what its seat promises. + DefinesSeats []SeatDeclaration `json:"seats,omitempty"` + + // Uses are seats this module sends to. It names the *seat*, never the module holding it, so + // the implementation can be replaced under it and no caller changes. A caller gets publish + // on that seat's inbound subjects and nothing else — not its outbound events, and not a + // subscription to the queue it writes to (design 29 §2). + Uses []string `json:"uses,omitempty"` + + // Tools are the tools this module answers — request and reply, awaited. + // + // **New, and not `serves`**, which this manifest already uses for the facts a consumer needs + // in order to reach a provision. Two meanings under one key would be a footgun in the one + // file a module author reads most. Until now a module's tools were known only at runtime, + // from MESH_TOOL_MODULES in its image; declaring them is what lets the mesh check that a + // module claiming a seat answers what that seat's protocol promises (novox/hq ADR 0118). + Tools []string `json:"tools,omitempty"` // Capabilities the machine must have. A different field from Requires because the remedy // differs: a missing module can be assigned, and a missing capability means the wrong @@ -424,6 +458,20 @@ type Manifest struct { // A directory rather than one document for the same reason as above: each value is sealed // separately and the mesh cannot open any of them to build a list. Grants map[string]string `json:"grants,omitempty"` + + // BusUsers is where this module wants the mesh's user list written, and it is only ever + // answered for the module holding `mesh-broker`. + // + // **The mesh writes who may connect; the module owns everything else about its server** + // (novox/hq design 25 §4, task 1.7). Ports, TLS paths and a store directory live in this + // module's image and its mounts and change when it does, so the module's own configuration + // carries them and includes this file. A controller that wrote the whole configuration would + // have to be kept in step with a Dockerfile it never sees. + // + // **Asking for it is not enough to receive it.** This file holds every user's password hash, so + // a module that could ask for it could read every credential on the bus — and the claim on + // `mesh-broker` is what authorises it, checked from this manifest alone. + BusUsers string `json:"bus-users,omitempty"` } // Build says how to produce this module's artifacts from its source. @@ -672,7 +720,22 @@ type Certificate struct { // CertificateID and AuthorityID are the resource identities of what the mesh issued. func CertificateID() string { return "certificate" } -func AuthorityID() string { return "certificate-authority" } + +// ClaimsSeat says whether this manifest claims one named seat. +func (m Manifest) ClaimsSeat(seat string) bool { + for _, c := range m.Claims { + if c.Name == seat { + return true + } + } + return false +} + +// BusUsersID names the mesh's composed user list, so it is the same resource across every +// declaration and a change to it is an update rather than a second file beside the old one — which +// on a bus reading a directory would be two account lists, and the server would take both. +func BusUsersID() string { return "bus-users" } +func AuthorityID() string { return "certificate-authority" } // FilteringID names the computed rule set, so it is the same resource across every declaration // and a change to it is an update rather than an addition beside the old one. @@ -1001,6 +1064,10 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, fmt.Sprintf("%s requires itself", m.Module)) } } + // What it may call an event, and what it may ask to hear (events.go). Checked here because a + // module whose event names are wrong installs, starts, connects and reacts to nothing, with + // every log line saying it is fine (novox/hq 04-ISSUES/127). + problems = append(problems, EventProblems(m)...) wellFormed := true for _, c := range m.Claims { if !name.MatchString(c.Name) { @@ -1021,6 +1088,10 @@ func ParseManifest(raw []byte) (Manifest, error) { if wellFormed { problems = append(problems, claimProblems(m)...) } + // What one manifest can be judged on: a declaration's shape, its scope, and the reserved + // prefix. Whether a seat anybody names exists, and whether a holder answers for it, are + // facts about the catalogue and are checked at registration (CatalogueProblems). + problems = append(problems, declaredSeatProblems(m)...) if m.Computed != "" && len(m.Resources) > 0 { // One or the other. A module that both ships files and has them computed would leave // nobody able to say where a given file came from. diff --git a/internal/catalogue/no_subjects_test.go b/internal/catalogue/no_subjects_test.go new file mode 100644 index 0000000..cb4b301 --- /dev/null +++ b/internal/catalogue/no_subjects_test.go @@ -0,0 +1,121 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// **A manifest holds no subject** (novox/hq design 29 §1). +// +// A module names its events, tools and seats locally, and the mesh derives where they land. The +// property that buys: reorganise the subject space and every manifest in the catalogue is still +// correct. It holds today by construction — nothing reads a subject from a manifest — and a rule +// held by construction is one a later field breaks quietly, with the symptom appearing as a +// permission that does not match a subject rather than as a manifest that was wrong. +func TestNoManifestContainsASubject(t *testing.T) { + root := filepath.Join("..", "..", "..", "mesh-catalog", "modules") + entries, err := os.ReadDir(root) + if err != nil { + t.Skipf("catalogue sibling not present: %v", err) + } + + // Anything in the mesh's own subject space, and anything shaped like a wire address. + subject := regexp.MustCompile(`^(mesh|\$JS)\.[a-zA-Z0-9_*>.-]+$`) + + var found []string + var walk func(module string, path string, v any) + walk = func(module, path string, v any) { + switch t := v.(type) { + case string: + if subject.MatchString(t) { + found = append(found, module+" "+path+" = "+t) + } + case map[string]any: + for k, inner := range t { + walk(module, path+"."+k, inner) + } + case []any: + for _, inner := range t { + walk(module, path+"[]", inner) + } + } + } + + checked := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json")) + if err != nil { + continue + } + var m any + if err := json.Unmarshal(raw, &m); err != nil { + t.Errorf("%s: %v", e.Name(), err) + continue + } + checked++ + walk(e.Name(), "", m) + } + if checked == 0 { + t.Skip("no manifests read") + } + if len(found) > 0 { + t.Errorf("a manifest names a subject, so reorganising the subject space would mean "+ + "editing the catalogue:\n %s", strings.Join(found, "\n ")) + } + t.Logf("%d manifests hold no subject", checked) +} + +// **Every module's event names are what design 29 says, across the whole catalogue.** +// +// The rule above holds by construction and turned out to be weaker than it reads: a manifest holds +// no subject, and every manifest in the catalogue still held the old bus's routing key, which +// derives into a namespace nobody owns (novox/hq 04-ISSUES/127). Nothing failed — the services +// started and none of them reacted. This is the check that was missing. +func TestEveryManifestsEventNamesAreLocal(t *testing.T) { + manifests := theCatalogue(t) + + var problems []string + for _, m := range manifests { + problems = append(problems, EventProblems(m)...) + } + if len(problems) > 0 { + t.Fatalf("the catalogue holds %d event name(s) the mesh would derive wrongly:\n %s", + len(problems), strings.Join(problems, "\n ")) + } +} + +// theCatalogue is every manifest beside this checkout, parsed the way registration parses one. +func theCatalogue(t *testing.T) []Manifest { + t.Helper() + root := filepath.Join("..", "..", "..", "mesh-catalog", "modules") + entries, err := os.ReadDir(root) + if err != nil { + t.Skipf("catalogue sibling not present: %v", err) + } + var out []Manifest + for _, e := range entries { + if !e.IsDir() { + continue + } + raw, err := os.ReadFile(filepath.Join(root, e.Name(), "module.json")) + if err != nil { + continue + } + var m Manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", e.Name(), err) + } + out = append(out, m) + } + if len(out) == 0 { + t.Skip("no manifests found beside this checkout") + } + return out +} diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 753bd30..8a70f70 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -27,6 +27,17 @@ type Seat struct { // provision may only be held by a module providing it at the seat's scope, and its holder is // what a requirement for that provision resolves to when several modules provide it. Delivers string + // Accepts, Emits and Serves are the protocol of the role, as local verbs — the same three a + // module declares for a seat of its own (novox/hq ADR 0118), and empty for most of these: a seat + // is usually about who does a job and not about what may be said to them. + // + // **Named here so the mesh has no role it cannot describe** (ADR 0121). Without them a build + // machine had three audiences for one outcome and nothing derived a grant for any of them, and an + // event about a role had nowhere to live but the namespace of whichever module held that role + // today — which the bus refuses, because a namespace belongs to who it is named for. + Accepts []string + Emits []string + Serves []string // Decision is the record that made it a seat. Decision string } @@ -39,7 +50,12 @@ type Seat struct { var defaultSeats = []Seat{ {Name: "mesh-controller", Scope: ScopeMesh, Decision: "novox/hq ADR 0079"}, {Name: "mesh-store", Scope: ScopeMesh, Delivers: "postgres-database", Decision: "novox/hq ADR 0079"}, - {Name: "mesh-broker", Scope: ScopeMesh, Delivers: "amqp", Decision: "novox/hq ADR 0079"}, + // **Delivers the mesh's own bus, not `amqp`.** Those were the same word until + // ADR 0127 separated them: `amqp` is a backing service a module may require, and this seat is + // the mesh's own transport. ADR 0128 then made that connection something a module requires + // rather than receives ambiently — 23 of the catalogue's modules never speak, and an ambient + // connection would mint a credential for each. + {Name: "mesh-broker", Scope: ScopeMesh, Delivers: "mesh-bus", Decision: "novox/hq ADR 0079"}, {Name: "the-artifact-store", Scope: ScopeMesh, Delivers: "artifact-store", Decision: "novox/hq ADR 0075"}, {Name: "mesh-catalog", Scope: ScopeMesh, Decision: "novox/hq ADR 0121"}, // Deferred renames (novox/hq ADR 0121): these deliver a provision, so renaming them is a @@ -47,7 +63,11 @@ var defaultSeats = []Seat{ // They keep their names until that migration is done deliberately, apart from the node-* pass. {Name: "npm-package-registry", Scope: ScopeMesh, Delivers: "npm-package-registry", Decision: "novox/hq ADR 0109"}, {Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0111"}, - {Name: "mesh-build-machine", Scope: ScopeMesh, Decision: "novox/hq ADR 0121"}, + // A build is work submitted to this role and its outcome is the role's own event (ADR 0129). + // One publish reaches whoever asked, the controller that records it, and the catalogue that + // places it in the graph — what the old bus's shared exchange did for free. + {Name: "mesh-build-machine", Scope: ScopeMesh, + Accepts: []string{"build"}, Emits: []string{"built"}, Decision: "novox/hq ADR 0121"}, {Name: "node-dns-resolver", Scope: ScopeNode, Decision: "novox/hq ADR 0121"}, {Name: "node-intrusion-prevention", Scope: ScopeNode, Decision: "novox/hq ADR 0121"}, {Name: "node-packet-filter", Scope: ScopeNode, Decision: "novox/hq ADR 0121"}, @@ -152,7 +172,7 @@ func SeatDelivering(provision string) (Seat, bool) { func claimProblems(m Manifest) []string { var problems []string - defined := map[string]Claim{} + defined := map[string]SeatDeclaration{} for _, d := range m.DefinesSeats { if _, isSystem := SeatNamed(d.Name); isSystem || isSystemSeatName(d.Name) { problems = append(problems, fmt.Sprintf( @@ -185,9 +205,12 @@ func claimProblems(m Manifest) []string { } d, ours := defined[c.Name] if !ours { - problems = append(problems, fmt.Sprintf( - "%s claims %q, which is not a seat this mesh defines and not one %s declares itself "+ - "(novox/hq ADR 0121) — the seats are: %s", m.Module, c.Name, m.Module, seatNames())) + // **A claim on a seat this manifest does not declare is not the parser's to judge.** + // A module may hold a seat another module declared — that is why ADR 0126 has callers + // name the seat and not its provider, so an implementation can be replaced without + // touching a caller. Whether the seat exists is a fact about the whole catalogue, so + // the refusal is at registration, where every declaration is in view + // (`CatalogueProblems`: "which no module declares and the mesh does not define"). continue } if c.At() != d.At() { @@ -243,3 +266,18 @@ func HolderAmong(provision string, providers []Provider, held []Held) (Provider, } return Provider{}, false } + +// SeatsWithAProtocol are the mesh's own seats that say something about what may be said to them or by +// them, which is the set the bus derives streams, consumers and permissions from. +// +// Most of the set is not here, and that is the ordinary case: a seat saying only who does a job grants +// nothing on the bus and needs no queue. +func SeatsWithAProtocol() []Seat { + var out []Seat + for _, s := range seats { + if len(s.Accepts) > 0 || len(s.Emits) > 0 || len(s.Serves) > 0 { + out = append(out, s) + } + } + return out +} diff --git a/internal/catalogue/seats_declared.go b/internal/catalogue/seats_declared.go new file mode 100644 index 0000000..5423c96 --- /dev/null +++ b/internal/catalogue/seats_declared.go @@ -0,0 +1,239 @@ +package catalogue + +import ( + "fmt" + "sort" + "strings" +) + +// Seats a module declares of its own (novox/hq ADR 0118). +// +// The set of seats a mesh has is **derived**: the mesh's own, in seats.go, plus those declared by +// every module it has registered. Still closed — a seat named nowhere is refused — but computed +// from the catalogue rather than written in the controller, which is what ADR 0110 actually +// needed and a hand-maintained table could not keep. Its own evidence: the enumeration done by +// hand while that record was written reported eleven claims where there were thirteen. +// +// **What can be checked from one manifest and what cannot.** A declaration's shape, its scope, +// and the reserved prefix are facts about the manifest in front of you. Whether a seat anybody +// names actually exists, whether two modules declared the same one, and whether a holder +// satisfies the protocol are facts about the *catalogue* — so they are checked at registration, +// by CatalogueProblems, which is the last moment the mesh can still say no. + +// meshSeatPrefix is reserved to the mesh. The prefix *is* the reservation rule: no list of +// reserved names to maintain, no way for the mesh's own namespace to be colonised by a manifest, +// and nothing to keep in step when a mesh seat is added. +const meshSeatPrefix = "mesh-" + +// A SeatDeclaration is a role a module offers on the bus: what may be sent to it, what it says, +// and what it answers. A caller declares that it uses the *seat*, never the module, so the +// implementation can be replaced under it. +type SeatDeclaration struct { + Name string `json:"name"` + Scope string `json:"scope,omitempty"` + + // Accepts are the verbs others may submit work on. Each becomes a work-queue subject, and + // the holder is the only consumer — so exactly one worker does the job, by construction + // rather than by how carefully somebody wrote a subscribe call. + Accepts []string `json:"accepts,omitempty"` + // Emits are the verbs the holder publishes: 1:many, nobody obliged to act. + Emits []string `json:"emits,omitempty"` + // Serves are the verbs the holder answers: request and reply, awaited. + Serves []string `json:"serves,omitempty"` + + // RetainSeconds is how long the inbound backlog survives with no holder, zero for the + // mesh's default. Retention belongs to whoever owns the namespace (design 29 §3) — a seat + // owns its own, which is why a seat is also the answer for a module that needs retention + // its events cannot have. + RetainSeconds int `json:"retain-seconds,omitempty"` +} + +// At is this declaration's scope, with the default applied. Mesh by default, because a seat +// declared by a module is nearly always "there is one of these in the mesh" — a per-node worker +// is the deliberate case, and says so. +func (s SeatDeclaration) At() string { + if s.Scope == "" { + return ScopeMesh + } + return s.Scope +} + +// verbs is everything the protocol names, for the checks that do not care which half. +func (s SeatDeclaration) verbs() []string { + out := append([]string{}, s.Accepts...) + out = append(out, s.Emits...) + return append(out, s.Serves...) +} + +// declaredSeatProblems is what one manifest can be judged on alone. +func declaredSeatProblems(m Manifest) []string { + var problems []string + seen := map[string]bool{} + + for _, s := range m.DefinesSeats { + switch { + case s.Name == "": + problems = append(problems, fmt.Sprintf("%s declares a seat with no name", m.Module)) + continue + case !name.MatchString(s.Name): + problems = append(problems, fmt.Sprintf( + "%s declares a seat named %q, which is not a usable name", m.Module, s.Name)) + continue + case strings.HasPrefix(s.Name, meshSeatPrefix): + // The mesh's own code dereferences its seats by name — the resolver *is* the thing + // that finds the store — so the prefix is not a convention, it is a namespace. + problems = append(problems, fmt.Sprintf( + "%s declares a seat named %q; %q is reserved to the mesh, which defines its own "+ + "seats (novox/hq ADR 0118)", m.Module, s.Name, meshSeatPrefix+"*")) + continue + } + if seen[s.Name] { + problems = append(problems, fmt.Sprintf( + "%s declares the seat %q twice", m.Module, s.Name)) + continue + } + seen[s.Name] = true + + if _, isMesh := SeatNamed(s.Name); isMesh { + problems = append(problems, fmt.Sprintf( + "%s declares %q, which is a seat the mesh already defines", m.Module, s.Name)) + } + switch s.At() { + case ScopeNode, ScopeSite, ScopeMesh: + default: + problems = append(problems, fmt.Sprintf( + "%s declares seat %s at scope %q; a seat is held per node, per site or per mesh", + m.Module, s.Name, s.Scope)) + } + // A seat with an empty protocol is allowed, and is the mesh saying what a machine is: + // which module is this node's packet filter, or its showcase. Design 26 calls it a seat + // that delivers nothing, and that is most of the node-scoped ones. ADR 0126's "a declared + // seat carries a protocol" governs what a holder must satisfy, not that every seat offers + // something — a marker seat's protocol is satisfied by holding it. Nothing can reach this + // state by accident: a mistyped field name is refused by the parser above, so an empty + // protocol was written as one. + for _, v := range s.verbs() { + if !name.MatchString(v) { + problems = append(problems, fmt.Sprintf( + "%s declares %s.%s, which is not a usable verb", m.Module, s.Name, v)) + } + } + } + + for _, u := range m.Uses { + if !name.MatchString(u) { + problems = append(problems, fmt.Sprintf("%s uses %q, which is not a usable seat name", m.Module, u)) + } + } + return problems +} + +// A Shelf is every manifest the mesh has registered, by module name. +type Shelf map[string]Manifest + +// CatalogueProblems are the rules no single manifest can be judged against. +// +// Run at registration, which is the last moment the mesh can still refuse: after it, a caller is +// bound to a seat and a refusal is an outage rather than a conversation. +func CatalogueProblems(shelf Shelf) []string { + var problems []string + + // Who declares what, and who declared it first. + declaredBy := map[string]string{} + declared := map[string]SeatDeclaration{} + for _, module := range shelfOrder(shelf) { + for _, s := range shelf[module].DefinesSeats { + if s.Name == "" { + continue + } + if first, taken := declaredBy[s.Name]; taken { + // The second loses. A seat name meaning two different protocols is the failure + // nobody could diagnose afterwards — a caller would bind to whichever happened + // to register first, and the symptom would appear in the other module. + problems = append(problems, fmt.Sprintf( + "%s declares the seat %q, which %s already declares; a seat name means one "+ + "protocol", module, s.Name, first)) + continue + } + declaredBy[s.Name] = module + declared[s.Name] = s + } + } + + exists := func(seat string) bool { + if _, isMesh := SeatNamed(seat); isMesh { + return true + } + _, ok := declaredBy[seat] + return ok + } + + for _, module := range shelfOrder(shelf) { + m := shelf[module] + + // A `uses` naming nothing is where ADR 0110's guarantee lands under a derived set: the + // same refusal, at the same moment, from a set nobody maintains by hand. + for _, u := range m.Uses { + if !exists(u) { + problems = append(problems, fmt.Sprintf( + "%s uses the seat %q, which no module declares and the mesh does not define", + module, u)) + } + } + + for _, c := range m.Claims { + if !exists(c.Name) { + problems = append(problems, fmt.Sprintf( + "%s claims the seat %q, which no module declares and the mesh does not define", + module, c.Name)) + continue + } + s, isModuleSeat := declared[c.Name] + if !isModuleSeat { + continue // a mesh seat: already judged by claimProblems + } + if c.At() != s.At() { + problems = append(problems, fmt.Sprintf( + "%s claims %s at scope %q, and %s declares it at %s", + module, c.Name, c.At(), declaredBy[c.Name], s.At())) + } + // A holder that does not answer what the seat promises is a caller's timeout, found + // at assignment instead. + if missing := unserved(m, s); len(missing) > 0 { + problems = append(problems, fmt.Sprintf( + "%s claims %s but does not serve %s, which that seat's protocol promises", + module, c.Name, strings.Join(missing, ", "))) + } + } + } + sort.Strings(problems) + return problems +} + +// unserved is what a seat's protocol promises and the claimant does not answer. Only the tools +// are checked: `accepts` and `emits` are wired by the runtime from the declaration, while a tool +// is code the module either has or has not written. +func unserved(m Manifest, s SeatDeclaration) []string { + has := map[string]bool{} + for _, t := range m.Tools { + has[t] = true + } + var missing []string + for _, t := range s.Serves { + if !has[t] { + missing = append(missing, t) + } + } + return missing +} + +// shelfOrder is the catalogue in a stable order, so two runs report the same problems in the same +// sequence — a refusal that reorders itself is a refusal nobody can diff. +func shelfOrder(shelf Shelf) []string { + out := make([]string, 0, len(shelf)) + for k := range shelf { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/catalogue/seats_declared_test.go b/internal/catalogue/seats_declared_test.go new file mode 100644 index 0000000..9c64d4b --- /dev/null +++ b/internal/catalogue/seats_declared_test.go @@ -0,0 +1,108 @@ +package catalogue + +import ( + "strings" + "testing" +) + +func problemsFor(t *testing.T, shelf Shelf) string { + t.Helper() + return strings.Join(CatalogueProblems(shelf), "; ") +} + +func telegram() Manifest { + return Manifest{Module: "telegram", Tools: []string{"status"}, DefinesSeats: []SeatDeclaration{{ + Name: "telegram-sender", Scope: ScopeMesh, + Accepts: []string{"send"}, Emits: []string{"delivered", "failed"}, Serves: []string{"status"}, + }}, Claims: []Claim{{Name: "telegram-sender", Scope: ScopeMesh}}} +} + +// The whole point: a module contributes a capability without the mesh being changed. +func TestAModuleDeclaresItsOwnSeatAndHoldsIt(t *testing.T) { + shop := Manifest{Module: "shop", Uses: []string{"telegram-sender"}} + if got := problemsFor(t, Shelf{"telegram": telegram(), "shop": shop}); got != "" { + t.Fatalf("a declared seat and its caller were refused: %s", got) + } +} + +// The prefix is the reservation rule, so there is no list to maintain and none to drift. +func TestAModuleCannotDeclareAMeshSeat(t *testing.T) { + for _, n := range []string{"mesh-broker", "mesh-anything", "mesh-store"} { + m := Manifest{Module: "impostor", DefinesSeats: []SeatDeclaration{{Name: n, Accepts: []string{"x"}}}} + got := strings.Join(declaredSeatProblems(m), "; ") + if !strings.Contains(got, "reserved to the mesh") { + t.Fatalf("%q was accepted as a module's seat: %q", n, got) + } + } +} + +// A seat name meaning two protocols is the failure nobody could diagnose afterwards. +func TestTwoModulesCannotDeclareTheSameSeat(t *testing.T) { + other := Manifest{Module: "aardvark", DefinesSeats: []SeatDeclaration{{ + Name: "telegram-sender", Scope: ScopeMesh, Accepts: []string{"something-else"}}}} + got := problemsFor(t, Shelf{"telegram": telegram(), "aardvark": other}) + if !strings.Contains(got, "already declares") { + t.Fatalf("both declarations stood: %s", got) + } + // The first declarer keeps it; only the second is refused. + if strings.Count(got, "already declares") != 1 { + t.Fatalf("expected exactly one refusal: %s", got) + } +} + +// Where ADR 0110's guarantee lands under a derived set: a typo is refused, not resolved to +// nothing at runtime. +func TestUsingASeatNobodyDeclaresIsRefused(t *testing.T) { + shop := Manifest{Module: "shop", Uses: []string{"telegram-sendr"}} + got := problemsFor(t, Shelf{"telegram": telegram(), "shop": shop}) + if !strings.Contains(got, "telegram-sendr") || !strings.Contains(got, "no module declares") { + t.Fatalf("a misspelled seat was accepted: %s", got) + } +} + +// A holder that does not answer what the seat promises is a caller's timeout, found here instead. +func TestAHolderMustServeWhatItsSeatPromises(t *testing.T) { + m := telegram() + m.Tools = nil // declares the seat, serves none of it + got := problemsFor(t, Shelf{"telegram": m}) + if !strings.Contains(got, "does not serve status") { + t.Fatalf("a holder was accepted that answers nothing its seat promises: %s", got) + } +} + +// A seat with no protocol is a marker: which module is this node's showcase, or its packet filter. +// Most node-scoped seats are markers, so refusing one would refuse the majority of the set. +func TestASeatWithoutAProtocolIsAMarkerNotAMistake(t *testing.T) { + m := Manifest{Module: "vague", DefinesSeats: []SeatDeclaration{{Name: "something", Scope: ScopeNode}}} + if got := strings.Join(declaredSeatProblems(m), "; "); got != "" { + t.Fatalf("a marker seat was refused: %s", got) + } +} + +// A claim at the wrong scope is a different seat than the one declared. +func TestAClaimMustMatchTheDeclaredScope(t *testing.T) { + m := telegram() + m.Claims = []Claim{{Name: "telegram-sender", Scope: ScopeNode}} + got := problemsFor(t, Shelf{"telegram": m}) + if !strings.Contains(got, "scope") { + t.Fatalf("a claim at the wrong scope was accepted: %s", got) + } +} + +// The mesh's own seats still work, and are not shadowed by the derived half. +func TestTheMeshsOwnSeatsAreStillClaimable(t *testing.T) { + m := Manifest{Module: "nats", Claims: []Claim{{Name: "mesh-broker", Scope: ScopeMesh}}} + if got := problemsFor(t, Shelf{"nats": m}); got != "" { + t.Fatalf("a mesh seat was refused by the derived check: %s", got) + } +} + +// A refusal that reorders itself between runs is a refusal nobody can diff. +func TestTheProblemsAreStable(t *testing.T) { + shelf := Shelf{"telegram": telegram(), "shop": {Module: "shop", Uses: []string{"nope"}}, + "other": {Module: "other", Uses: []string{"also-nope"}}} + first, second := problemsFor(t, shelf), problemsFor(t, shelf) + if first != second { + t.Fatalf("unstable:\n%s\n%s", first, second) + } +} diff --git a/internal/catalogue/seats_test.go b/internal/catalogue/seats_test.go index f479953..030d043 100644 --- a/internal/catalogue/seats_test.go +++ b/internal/catalogue/seats_test.go @@ -74,17 +74,40 @@ func claimed(claims string) []byte { return []byte(`{"module":"thing","version":"1","provides":[{"name":"npm-package-registry","scope":"mesh"}],"claims":` + claims + `}`) } -func TestAClaimOnASeatTheMeshDoesNotDefineIsRefused(t *testing.T) { - _, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) - if err == nil { - t.Fatal("a module invented a seat by claiming it") +// **The refusal moved, it did not go** (novox/hq ADR 0118, superseding 0110). A module may now +// declare its own seats, so whether a claimed seat exists is a fact about the *catalogue* and not +// about the manifest in front of the parser: a claim on a seat another registered module declares +// is perfectly good, and the parser cannot tell the two cases apart. So the parser accepts it and +// registration refuses it — the same guarantee, at the same moment work would otherwise start, +// from a set nobody maintains by hand. +func TestAClaimOnASeatNobodyDeclaresIsRefusedAtRegistration(t *testing.T) { + m, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) + if err != nil { + t.Fatalf("the parser judged a claim it cannot judge alone: %v", err) } - if !strings.Contains(err.Error(), "the-anything") || !strings.Contains(err.Error(), "not a seat") { - t.Fatalf("the refusal does not say the seat is unknown: %v", err) + + problems := CatalogueProblems(Shelf{m.Module: m}) + if len(problems) == 0 { + t.Fatal("a module invented a seat by claiming it, and registration allowed it") } - // And it says what the seats are, because "no" without the list sends somebody reading code. - if !strings.Contains(err.Error(), "node-packet-filter") { - t.Fatalf("the refusal does not list the seats: %v", err) + joined := strings.Join(problems, "; ") + if !strings.Contains(joined, "the-anything") || !strings.Contains(joined, "no module declares") { + t.Fatalf("the refusal does not say the seat is nobody's: %v", problems) + } +} + +// And the same claim is fine once something declares that seat, which is the case the parser +// could not have distinguished. +func TestAClaimOnASeatAnotherModuleDeclaresIsAccepted(t *testing.T) { + claimant, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) + if err != nil { + t.Fatal(err) + } + declarer := Manifest{Module: "someone", DefinesSeats: []SeatDeclaration{ + {Name: "the-anything", Scope: ScopeNode, Accepts: []string{"work"}}, + }} + if problems := CatalogueProblems(Shelf{claimant.Module: claimant, "someone": declarer}); len(problems) != 0 { + t.Fatalf("a claim on a declared seat was refused: %v", problems) } } @@ -96,8 +119,15 @@ func TestAModuleDefinesAndClaimsItsOwnSeat(t *testing.T) { if _, err := ParseManifest(ok); err != nil { t.Fatalf("a module could not define and claim its own seat: %v", err) } - // Claiming a name it neither the mesh nor the module defines is still refused. - if _, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)); err == nil { + // Claiming a name nobody defines is still refused — but **at registration, not here**: with + // seats declared by modules, a claim on a seat *another* module declares is good, and the + // parser cannot tell that from an invented name. See + // TestAClaimOnASeatNobodyDeclaresIsRefusedAtRegistration. + claimant, err := ParseManifest(claimed(`[{"name":"the-anything","scope":"node"}]`)) + if err != nil { + t.Fatalf("the parser judged a claim it cannot judge alone: %v", err) + } + if len(CatalogueProblems(Shelf{claimant.Module: claimant})) == 0 { t.Fatal("a module claimed a seat nobody defines") } // A module may not carve its seat out of the mesh's own namespace. @@ -271,11 +301,11 @@ func TestUseSeatsReplacesTheSetButNeverEmptiesIt(t *testing.T) { // and a held record naming the old name break nothing after a rename. func TestAFormerNameResolvesAfterARename(t *testing.T) { defer func() { UseSeats(DefaultSeats()); UseAliases(nil) }() - UseSeats([]Seat{{Name: "mesh-git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0121"}}) - UseAliases(map[string]string{"git": "mesh-git"}) + UseSeats([]Seat{{Name: "git", Scope: ScopeMesh, Delivers: "git", Decision: "novox/hq ADR 0121"}}) + UseAliases(map[string]string{"git": "git"}) // The old name resolves to the renamed seat. - if s, ok := SeatNamed("git"); !ok || s.Name != "mesh-git" { + if s, ok := SeatNamed("git"); !ok || s.Name != "git" { t.Fatalf("the former name did not resolve to the renamed seat: %+v ok=%v", s, ok) } // And a holder recorded under the old name is still found for the provision the seat delivers. diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go new file mode 100644 index 0000000..4472269 --- /dev/null +++ b/internal/inventory/busrecords.go @@ -0,0 +1,178 @@ +package inventory + +import ( + "context" + "fmt" + "strings" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// What the bus's user list is derived from, read out of the mesh's records. +// +// The deriving itself is pure and lives in the broker package; this is the reading, and it is kept +// apart for the reason that package keeps its own types: a permission must be a function of what a +// module declared, and a query that decided anything would be a second place authority came from. + +// BusRecords is every fact the composer needs about who may reach the bus. +// +// **A module's authority comes from the manifest, not from the assignment.** The assignment says +// *where* it runs; what it may say is in what it declared, so the two are read together and the +// manifest is the one that decides. +func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) { + nodes, err := i.Nodes(ctx) + if err != nil { + return broker.Records{}, fmt.Errorf("cannot read the mesh's machines: %w", err) + } + declared, err := i.Catalogue(ctx) + if err != nil { + return broker.Records{}, fmt.Errorf("cannot read the catalogue: %w", err) + } + + // Every seat any module declares, by name, so a module's claim can be resolved to the protocol + // that seat promises. **Across the whole catalogue, not one manifest**: a seat is declared by + // one module and held by another, which is the whole reason a seat exists (ADR 0118). + seats := map[string]catalogue.SeatDeclaration{} + for _, m := range declared { + for _, s := range m.DefinesSeats { + seats[s.Name] = s + } + } + // And the mesh's own, which carry protocol too (novox/hq ADR 0121). Added after the modules' + // rather than before, because a `mesh-*` name is the mesh's and registration refuses a module + // declaring one — so this cannot be shadowed, and if it ever were, the mesh's own would win. + for _, own := range catalogue.SeatsWithAProtocol() { + seats[own.Name] = catalogue.SeatDeclaration{ + Name: own.Name, Scope: own.Scope, + Accepts: own.Accepts, Emits: own.Emits, Serves: own.Serves, + } + } + + out := broker.Records{Assigned: map[string][]broker.Declared{}, People: map[string][]string{}} + for _, n := range nodes { + out.Nodes = append(out.Nodes, n.Name) + modules, err := i.Assigned(ctx, n.Name) + if err != nil { + return broker.Records{}, fmt.Errorf("cannot read what %s runs: %w", n.Name, err) + } + for _, module := range modules { + m, known := declared[module] + if !known { + // Assigned and not in the catalogue. Said rather than composed with no authority: + // a user with an empty permission list is a module that starts, connects, and is + // refused by the server on its first publish — an authorisation error that says + // nothing about a missing manifest. + // + // **The catalogue refuses to forget an assigned module, so this is the second line + // and not the first.** It earns its place there anyway: relying on another + // package's invariant is how a rule ends up enforced by nothing. + return broker.Records{}, fmt.Errorf( + "%s is assigned to %s and is not in the catalogue, so what it may say cannot "+ + "be derived", module, n.Name) + } + out.Assigned[n.Name] = append(out.Assigned[n.Name], declaredFor(m, seats)) + } + } + + enrolling, err := i.NodesWithALiveToken(ctx) + if err != nil { + return broker.Records{}, err + } + out.Enrolling = enrolling + + people, err := i.People(ctx) + if err != nil { + return broker.Records{}, err + } + for _, p := range people { + out.People[p.Name] = p.Invokes + } + return out, nil +} + +// declaredFor is one module's manifest as the composer needs it: what it says about itself, and the +// protocol of every seat it holds or uses. +func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) broker.Declared { + // A consumed name is a module's event unless it names a seat, and only somebody holding the seat + // set can tell (novox/hq ADR 0121). Split here, because the composer cannot look at a name and + // know — and a role's event read as a module's is a subscription to a namespace nobody owns. + var fromModules []string + var watches []broker.Seat + for _, c := range m.Consumes { + emitter, event, named := strings.Cut(c, ".") + if named { + if s, isASeat := seats[emitter]; isASeat { + watches = append(watches, broker.Seat{Name: s.Name, Emits: []string{event}}) + continue + } + } + fromModules = append(fromModules, c) + } + + d := broker.Declared{ + Module: m.Module, + Emits: m.Emits, + Consumes: fromModules, + Watches: watches, + // The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the + // facts a consumer needs to reach a provision, a different meaning under a similar word. + Serves: m.Tools, + } + for _, c := range m.Claims { + // Every seat with a protocol, the mesh's own included. One that says only who does a job is + // not here and grants nothing, which is most of them. + if s, hasAProtocol := seats[c.Name]; hasAProtocol { + d.Holds = append(d.Holds, asSeat(s)) + } + } + for _, name := range m.Uses { + if s, declaredSomewhere := seats[name]; declaredSomewhere { + d.Uses = append(d.Uses, asSeat(s)) + } + } + return d +} + +func asSeat(s catalogue.SeatDeclaration) broker.Seat { + return broker.Seat{Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves} +} + +// MeshSeats are the mesh's own seats that carry a protocol, as the bus needs them: what to make a work +// queue for, and whose holder gets a worker on it (novox/hq ADR 0121). +func MeshSeats() []broker.DeclaredSeat { + var out []broker.DeclaredSeat + for _, s := range catalogue.SeatsWithAProtocol() { + out = append(out, broker.DeclaredSeat{ + Name: s.Name, Accepts: s.Accepts, Emits: s.Emits, Serves: s.Serves, + }) + } + return out +} + +// NodesWithALiveToken is every machine holding a token that could still be presented — issued, not +// expired, not redeemed. +// +// **One enrolment user per such token** (design 25 §6): the inbox an answer goes to is scoped to the +// token, because an answer carries that machine's credentials sealed to it and a shared inbox is one +// machine able to read another's. +func (i *Inventory) NodesWithALiveToken(ctx context.Context) ([]string, error) { + rows, err := i.store.Pool().Query(ctx, + `select distinct n.name + from enrolment_token t join node n on n.id = t.node + where t.redeemed is null and t.expires > now() + order by n.name`) + if err != nil { + return nil, fmt.Errorf("cannot read which machines hold a live token: %w", err) + } + defer rows.Close() + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} diff --git a/internal/inventory/busrecords_test.go b/internal/inventory/busrecords_test.go new file mode 100644 index 0000000..24185e3 --- /dev/null +++ b/internal/inventory/busrecords_test.go @@ -0,0 +1,164 @@ +package inventory + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// Reading the bus's user list out of the mesh's records, against a real store. +// +// What each of these is about is a user that would be **missing or wrong in a way nothing reports**: +// the server reads whatever file it is given, and a module whose user is absent fails on its first +// publish with an authorisation error that says nothing about a missing assignment. + +func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) { + t.Helper() + inv := ForTest(t) + ctx := context.Background() + for _, m := range manifests { + if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + return inv, ctx +} + +func theSeatDeclarer() catalogue.Manifest { + return catalogue.Manifest{ + Module: "telegram", Version: "1", + DefinesSeats: []catalogue.SeatDeclaration{{ + Name: "telegram-sender", Accepts: []string{"send"}, Emits: []string{"delivered"}, + }}, + Claims: []catalogue.Claim{{Name: "telegram-sender", Scope: catalogue.ScopeMesh}}, + } +} + +// A module assigned to a machine becomes a user with the authority its manifest declared — and the +// protocol of a seat declared by a *different* module, which is the whole reason a seat exists. +func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) { + shop := catalogue.Manifest{ + Module: "shop", Version: "1", + Emits: []string{"order.placed"}, Tools: []string{"price"}, + Uses: []string{"telegram-sender"}, + } + inv, ctx := aMeshWith(t, theSeatDeclarer(), shop) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "one", "shop"); err != nil { + t.Fatal(err) + } + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + on := records.Assigned["one"] + if len(on) != 1 || on[0].Module != "shop" { + t.Fatalf("the machine's modules read as %+v", on) + } + if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" { + t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+ + "seat it was assigned to send to", on[0].Uses) + } + if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" { + t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+ + "serves nothing", on[0].Serves) + } + + // And it derives into a user the server would accept. + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var found bool + for _, u := range users { + if u.Username() != "one.shop" { + continue + } + found = true + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") || + !granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") || + !granted(perms.Subscribe, "mesh.mod.shop.tool.price") { + t.Fatalf("one.shop's authority is not what it declared: %+v", perms) + } + } + if !found { + t.Fatal("no user was derived for the assigned module") + } +} + +// A machine holding a live token gets an enrolment user; one whose token is spent or expired does +// not. **An enrolment user outliving its token is a right to join that nobody issued.** +func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) { + inv, ctx := aMeshWith(t) + for _, name := range []string{"live", "expired", "none"} { + if _, err := inv.AddNode(ctx, name); err != nil { + t.Fatal(err) + } + } + if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil { + t.Fatal(err) + } + // Briefly, then waited out: a token with no lifetime is refused at issue, which is the right + // refusal and leaves this as the way to have an expired one. + if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if strings.Join(records.Enrolling, ",") != "live" { + t.Fatalf("machines with a live token read as %v", records.Enrolling) + } +} + +// A module assigned and absent from the catalogue is refused rather than composed with no authority. +// +// **The catalogue refuses to forget an assigned module, so this is the second line and not the +// first** — and it earns its place there: relying on another package's invariant is how a rule ends +// up enforced by nothing. Checked against the derivation directly, because the situation cannot be +// reached through the store. +func TestAnAssignmentWithNoManifestDerivesNoAuthority(t *testing.T) { + // What BusRecords would have produced had it composed a ghost: a module with nothing declared. + users, err := broker.Users(broker.Records{ + Nodes: []string{"one"}, + Assigned: map[string][]broker.Declared{"one": {{Module: "ghost"}}}, + }) + if err != nil { + t.Fatal(err) + } + perms, err := broker.PermissionsFor(users[len(users)-1]) + if err != nil { + t.Fatal(err) + } + // Its inbox and its ack subject, and nothing it could say. That is a module which starts, + // connects, and is refused by the server on its first publish — an authorisation error that + // says nothing about a missing manifest, which is why BusRecords names it instead. + for _, p := range perms.Publish { + if strings.HasPrefix(p, "mesh.mod.ghost.event.") { + t.Fatalf("a module with no manifest was granted %s", p) + } + } +} + +func granted(all []string, one string) bool { + for _, s := range all { + if s == one { + return true + } + } + return false +} diff --git a/internal/inventory/bususers.go b/internal/inventory/bususers.go new file mode 100644 index 0000000..b46a643 --- /dev/null +++ b/internal/inventory/bususers.go @@ -0,0 +1,232 @@ +package inventory + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "golang.org/x/crypto/bcrypt" +) + +// The bus's own users, as records. +// +// **Only the credential is kept here.** A user's *authority* is derived from what its module +// declares, every time the file is written (novox/hq ADR 0043) — a stored copy of a permission list +// would be a second account of a user's authority, able to disagree with the first, and the +// disagreement would be invisible until somebody compared a composed file with a manifest. +// +// What cannot be derived is the password, and on the bus being built it has to outlive its own +// minting: the whole user list is one file, rewritten whenever any of it changes, so a person's +// access change would blank every module's password if the mesh kept nothing (design 25 §4, and the +// migration beside this). + +// BusUser is one user of the bus, as the mesh records it. +type BusUser struct { + Username string + Kind string + Node string + Module string + // PasswordHash is what the composed file carries. The plaintext is returned once, by Mint, and + // then exists only where it was sealed. + PasswordHash string +} + +// The kinds of bus user the mesh records. The same words the composer uses, so a row and a +// principal do not need a translation table between them. +const ( + BusController = "controller" + BusNode = "node" + BusModule = "module" + BusEnrolment = "enrolment" + BusPerson = "person" +) + +// MintBusPassword makes a bus password and records its hash under a username, replacing whatever was +// there, and returns the plaintext **once**. +// +// **Once is the whole contract.** The caller seals it to whoever will use it — into an enrolment +// reply, into a module's sealed environment — and the mesh keeps only the hash, so a credential is +// never recoverable from the store. A caller that loses it must mint again, which is a rotation and +// is meant to feel like one. +func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) { + if u.Username == "" || u.Kind == "" { + return "", errors.New("a bus user needs a username and a kind") + } + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", fmt.Errorf("cannot generate a bus password: %w", err) + } + password := base64.RawURLEncoding.EncodeToString(raw) + + // The cost the server will pay on every connection. Left at the library's default rather than + // raised: a node reconnecting after a network blip pays it, and the mesh's own links reconnect + // far more often than a person logs in anywhere. + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", fmt.Errorf("cannot hash a bus password: %w", err) + } + + if _, err := i.store.Pool().Exec(ctx, + `insert into bus_user (username, kind, node, module, password_hash) + values ($1, $2, $3, $4, $5) + on conflict (username) do update + set kind = excluded.kind, node = excluded.node, module = excluded.module, + password_hash = excluded.password_hash, minted_at = now()`, + u.Username, u.Kind, u.Node, u.Module, string(hash)); err != nil { + return "", fmt.Errorf("cannot record the bus user %s: %w", u.Username, err) + } + return password, nil +} + +// BusUsers is every user the composed file should contain, by username. +// +// Returned as a map because the composer asks by username: the principals are derived from records +// elsewhere, and this is only what each one's password is. A principal with no row here has no +// password, and the composer refuses it rather than writing a user anybody is. +func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) { + rows, err := i.store.Pool().Query(ctx, + `select username, kind, node, module, password_hash from bus_user order by username`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]BusUser{} + for rows.Next() { + var u BusUser + if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash); err != nil { + return nil, err + } + out[u.Username] = u + } + return out, rows.Err() +} + +// BusUserHash is one user's hash, or false when the mesh has never minted one for it. +func (i *Inventory) BusUserHash(ctx context.Context, username string) (string, bool, error) { + var hash string + err := i.store.Pool().QueryRow(ctx, + `select password_hash from bus_user where username = $1`, username).Scan(&hash) + if errors.Is(err, pgx.ErrNoRows) { + return "", false, nil + } + return hash, err == nil, err +} + +// ForgetBusUser removes one user, so the next composition does not contain it. +// +// **Removal is what makes revocation real here.** On a bus with a management call, deleting an +// account ends its connections; here the credential stops working when the file no longer names it, +// which is the next composition — so forgetting the row and composing are one act, and a caller +// that does the first without the second has revoked nothing. +func (i *Inventory) ForgetBusUser(ctx context.Context, username string) error { + _, err := i.store.Pool().Exec(ctx, `delete from bus_user where username = $1`, username) + return err +} + +// ForgetBusUsersOf removes every user belonging to one node — its host's, and every module assigned +// to it. What a forgotten node leaves behind on the bus is otherwise a set of credentials for a +// machine the mesh no longer knows. +func (i *Inventory) ForgetBusUsersOf(ctx context.Context, node string) error { + if node == "" { + return errors.New("forgetting the bus users of no node would forget every user that has none") + } + _, err := i.store.Pool().Exec(ctx, `delete from bus_user where node = $1`, node) + return err +} + +// SeedBusUser records a hash of a credential the mesh did not mint, so a composition contains it. +// +// **Genesis is the reason this exists.** The controller's own user is created before the controller +// runs — by the installer, at a well-known bootstrap password, the way the store's and the old bus's +// are (`postgres:bootstrap`, `guest:guest`). Nothing minted it, so nothing recorded a hash for it, and +// the controller's first composition would leave itself out of the very file it was writing: a bus +// nothing can connect to, produced by the thing connected to it. +// +// Idempotent, and it does not overwrite. A credential the mesh *did* mint is the one that counts, so +// once there is a row this does nothing — otherwise a restart would put the bootstrap password back +// over a rotated one. +func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string) error { + if u.Username == "" || u.Kind == "" || password == "" { + return errors.New("a bus user needs a username, a kind and the credential it is using") + } + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return fmt.Errorf("cannot hash a bus password: %w", err) + } + _, err = i.store.Pool().Exec(ctx, + `insert into bus_user (username, kind, node, module, password_hash) + values ($1, $2, $3, $4, $5) + on conflict (username) do nothing`, + u.Username, u.Kind, u.Node, u.Module, string(hash)) + return err +} + +// A person who may call the mesh's tools (novox/hq design 25 §7). +// +// **Their authority is a list of tools and nothing else.** Not a module: they hold no seat, nothing is +// addressed to them, nothing is delivered to them, and they have no consumer to acknowledge. What +// they have is permission to ask. + +// Person is somebody who may reach the mesh's tools. +type Person struct { + Name string + // Invokes are the tools they may call, each `.`, or the single entry `*` for an + // administrator. + Invokes []string +} + +// RecordPerson adds somebody, or changes what they may call. +// +// Replacing rather than merging: what a person may call is stated in full, so a change that meant to +// remove a tool does remove it. A list that could only grow is a permission nobody can take back. +func (i *Inventory) RecordPerson(ctx context.Context, p Person) error { + if p.Name == "" { + return errors.New("a person needs a name: it becomes their user on the bus") + } + if len(p.Invokes) == 0 { + return fmt.Errorf( + "%s may call nothing, so there is no reason for them to reach the mesh. Name the tools, "+ + "or `*` for an administrator", p.Name) + } + _, err := i.store.Pool().Exec(ctx, + `insert into person (name, invokes) values ($1, $2) + on conflict (name) do update set invokes = excluded.invokes`, + p.Name, p.Invokes) + return err +} + +// People is everybody who may reach the mesh's tools. +func (i *Inventory) People(ctx context.Context) ([]Person, error) { + rows, err := i.store.Pool().Query(ctx, `select name, invokes from person order by name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Person + for rows.Next() { + var p Person + if err := rows.Scan(&p.Name, &p.Invokes); err != nil { + return nil, err + } + out = append(out, p) + } + return out, rows.Err() +} + +// ForgetPerson removes somebody and the credential they were given. +// +// **Both, or neither is a revocation.** A person's row gone and their bus user left behind is a +// credential that still works and that nothing derives, which is the worst of both: it keeps working +// and nobody can explain why. +func (i *Inventory) ForgetPerson(ctx context.Context, name string) error { + if name == "" { + return errors.New("forgetting nobody would forget everybody") + } + if _, err := i.store.Pool().Exec(ctx, `delete from person where name = $1`, name); err != nil { + return err + } + return i.ForgetBusUser(ctx, "person."+name) +} diff --git a/internal/inventory/bususers_test.go b/internal/inventory/bususers_test.go new file mode 100644 index 0000000..199b660 --- /dev/null +++ b/internal/inventory/bususers_test.go @@ -0,0 +1,160 @@ +package inventory + +import ( + "context" + "testing" + + "golang.org/x/crypto/bcrypt" +) + +// The bus's users as records — against a real store, because what is being checked is that the +// column exists, the upsert behaves, and a plaintext is returned exactly once. + +func aBusUser(module string) BusUser { + return BusUser{Username: "one." + module, Kind: BusModule, Node: "one", Module: module} +} + +// The plaintext comes back once and the store keeps only a hash that verifies against it. **A +// credential recoverable from the mesh's store is one whose blast radius is the store's**, so what +// is asserted is that the password is not in there. +func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + password, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if password == "" { + t.Fatal("no password came back, so nothing can be sealed to the module") + } + + hash, known, err := inv.BusUserHash(ctx, "one.shop") + if err != nil || !known { + t.Fatalf("the user was not recorded: %v %v", known, err) + } + if hash == password { + t.Fatal("the store holds the password itself") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil { + t.Fatalf("the recorded hash does not verify the password it was made from: %v", err) + } +} + +// Minting again replaces what was there rather than failing or adding a second row: that is a +// rotation, and the old credential stops working at the next composition. +func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + first, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + second, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("minting twice produced the same password") + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if len(users) != 1 { + t.Fatalf("%d users after two mints for one name", len(users)) + } + hash := users["one.shop"].PasswordHash + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil { + t.Fatal("the kept hash is not the newest password's") + } + if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil { + t.Fatal("the previous password still verifies, so a rotation revoked nothing") + } +} + +// Forgetting a node takes every credential that belonged to it — its host's and every module +// assigned to it. What a forgotten node leaves behind otherwise is a working set of credentials for +// a machine the mesh no longer knows. +func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + for _, u := range []BusUser{ + {Username: "node.one", Kind: BusNode, Node: "one"}, + aBusUser("shop"), + {Username: "node.two", Kind: BusNode, Node: "two"}, + {Username: "controller", Kind: BusController}, + } { + if _, err := inv.MintBusPassword(ctx, u); err != nil { + t.Fatal(err) + } + } + if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil { + t.Fatal(err) + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := users["node.one"]; still { + t.Fatal("a forgotten node's host credential still works") + } + if _, still := users["one.shop"]; still { + t.Fatal("a module on a forgotten node still has a credential") + } + // And nothing else went with it: the controller has no node, and another machine's user is + // another machine's. + for _, kept := range []string{"node.two", "controller"} { + if _, ok := users[kept]; !ok { + t.Fatalf("%s was removed with another node's users", kept) + } + } +} + +// Forgetting the users of no node would forget every user that has none — the controller and every +// person — so it is refused rather than run. +func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { + inv := ForTest(t) + if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil { + t.Fatal("forgetting the bus users of no node was allowed") + } +} + +// The controller's own user is created by the installer, so the mesh has to be able to record a +// credential it did not mint — or the first composition leaves the writer out of the file it writes. +func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, known, err := inv.BusUserHash(ctx, "controller") + if err != nil || !known { + t.Fatalf("the credential was not recorded: %v %v", known, err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { + t.Fatalf("what was recorded does not verify the credential given: %v", err) + } + + // Minted since, then seeded again — which is what a restart does. The rotation must stand, or + // every restart would put the bootstrap password back over it. + minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController}) + if err != nil { + t.Fatal(err) + } + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, _, err = inv.BusUserHash(ctx, "controller") + if err != nil { + t.Fatal(err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil { + t.Fatal("a restart put the bootstrap credential back over a rotated one") + } +} diff --git a/internal/inventory/migrations/0037-the-bus-keeps-its-users-hashes.sql b/internal/inventory/migrations/0037-the-bus-keeps-its-users-hashes.sql new file mode 100644 index 0000000..0ac7ee0 --- /dev/null +++ b/internal/inventory/migrations/0037-the-bus-keeps-its-users-hashes.sql @@ -0,0 +1,36 @@ +-- Every bus user's password hash, because the file has to be written again. +-- +-- novox/hq design 25 §4, task 1.7. On the bus the mesh runs on today an account is created by a +-- management call: the mesh mints a password, hands it over, seals the plaintext to whoever will +-- use it, and keeps nothing. That works because the broker remembers. +-- +-- The bus being built has no management call — its users are a file the controller composes, and +-- **the whole file is written every time any of it changes**. So the first person's access change +-- would silently blank every module's password. The hash has to outlive its own minting, which is +-- state the mesh did not need before and does now. +-- +-- Keyed by username, because the username is exactly what the composed file needs and what a +-- principal derives from its own identity. Nothing else about the user is here: **permissions are +-- not stored.** They are derived from what each module declares, every time the file is written +-- (ADR 0043) — a stored copy would be a second account of a user's authority, able to disagree +-- with the first, and the disagreement would be invisible until somebody compared a file with a +-- manifest. +-- +-- The hash and not the password. A file on a node's disk holds the hash, and so does this: a +-- credential recoverable from the mesh's store is one whose blast radius is the store's. +create table bus_user ( + username text primary key, + -- kind and what it names, so a user whose subject is gone can be found and removed: a module + -- unassigned, a node forgotten, a token spent. Recorded rather than parsed back out of the + -- username, because a name is for the server and a parser over it would be a second grammar. + kind text not null, + node text not null default '', + module text not null default '', + password_hash text not null, + minted_at timestamptz not null default now() +); + +-- Finding every user of one kind, and every user belonging to one node — which is what removing a +-- node, or composing after an assignment, asks. +create index bus_user_kind on bus_user (kind); +create index bus_user_node on bus_user (node) where node <> ''; diff --git a/internal/inventory/migrations/0038-a-person-may-reach-the-mesh.sql b/internal/inventory/migrations/0038-a-person-may-reach-the-mesh.sql new file mode 100644 index 0000000..83b3082 --- /dev/null +++ b/internal/inventory/migrations/0038-a-person-may-reach-the-mesh.sql @@ -0,0 +1,19 @@ +-- A person who may call the mesh's tools from a workstation. +-- +-- novox/hq design 25 §7. Everything else that reaches the bus is a machine or a module running on +-- one; this is the exception the mesh has always had informally — somebody at a terminal — and never +-- recorded. Until now "the operator" meant whoever held the keys, which is a role and not a record, +-- so nothing could say who may call what. +-- +-- **The authority is a list of tools and nothing else.** A person is not a module: they hold no seat, +-- nothing is addressed to them, nothing is delivered to them, and they have no consumer to +-- acknowledge. What they have is permission to ask. That is why there is no scope column and no node +-- column — a person is not on a machine. +create table person ( + name text primary key, + -- The tools this person may invoke, each `.`, or the single entry `*` for an + -- administrator. Stored as given: the permission is derived from it at every composition, so a + -- normalised form here would be a second opinion about authority (novox/hq ADR 0043). + invokes text[] not null default '{}', + created timestamptz not null default now() +); diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 913c383..5dfb5e6 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -821,6 +821,25 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error { return err } +// Outstanding is the digest of the declaration a machine was last sent, by its name, and empty +// for one that has never been sent anything. +// +// **By name rather than by id**, because the caller is the serving loop and what a node puts in a +// report is its name. Asked of one machine rather than read from Waiting's sweep, because it is +// asked per message: a report names the declaration it is about, and a report about one the mesh +// has already moved past is not acted on (design 25 §3). +func (i *Inventory) Outstanding(ctx context.Context, name string) (string, error) { + var sent string + err := i.store.Pool().QueryRow(ctx, + `select coalesce(sent, '') from node where name = $1`, name).Scan(&sent) + if errors.Is(err, pgx.ErrNoRows) { + // Not an error worth carrying up: a report from a machine the mesh has no record of has + // nothing to be stale against, and whatever is wrong with it is the listener's to say. + return "", nil + } + return sent, err +} + // Waiting is every machine whose declaration has changed since it was last sent one. // // The caller works out what each machine should be now, because only it can — resolution is the diff --git a/internal/inventory/people_test.go b/internal/inventory/people_test.go new file mode 100644 index 0000000..1085d57 --- /dev/null +++ b/internal/inventory/people_test.go @@ -0,0 +1,120 @@ +package inventory + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +// Somebody who may call the mesh's tools, and what the bus makes of them. + +// A person's authority is a list of tools, and it becomes exactly that on the bus — nothing on +// control, nothing on nodes, nothing they could publish as a module. +func TestAPersonMayCallToolsAndNothingElse(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + if err := inv.RecordPerson(ctx, Person{Name: "ada", + Invokes: []string{"mesh-catalog.catalog_tools"}}); err != nil { + t.Fatal(err) + } + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if got := records.People["ada"]; len(got) != 1 || got[0] != "mesh-catalog.catalog_tools" { + t.Fatalf("ada may call %v", got) + } + + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var found bool + for _, u := range users { + if u.Username() != "person.ada" { + continue + } + found = true + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if len(perms.Publish) != 1 || perms.Publish[0] != "mesh.mod.mesh-catalog.tool.catalog_tools" { + t.Errorf("ada may publish %v, which should be the one tool and nothing else", perms.Publish) + } + for _, s := range perms.Publish { + if strings.HasPrefix(s, "mesh.control") || strings.HasPrefix(s, "mesh.node") || + strings.Contains(s, ".event.") { + t.Errorf("a person may publish %s — an event would let them claim a module said "+ + "something, and control is not theirs", s) + } + } + if perms.AllowResponses { + t.Error("a person may answer a request, which is impersonating a module on a bus where " + + "anyone may serve a tool") + } + } + if !found { + t.Fatal("no bus user was derived for a recorded person") + } +} + +// Stating what somebody may call replaces what was there. A list that could only grow is a permission +// nobody can take back. +func TestChangingWhatAPersonMayCallRemovesWhatIsNotNamed(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one", "b.two"}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one"}}); err != nil { + t.Fatal(err) + } + people, err := inv.People(ctx) + if err != nil { + t.Fatal(err) + } + if len(people) != 1 || len(people[0].Invokes) != 1 || people[0].Invokes[0] != "a.one" { + t.Fatalf("ada may call %v; the removed tool is still there", people) + } +} + +// Somebody who may call nothing is refused: there is no reason for them to reach the mesh, and an +// empty list is more likely a mistake than an intention. +func TestSomebodyWhoMayCallNothingIsRefused(t *testing.T) { + inv := ForTest(t) + if err := inv.RecordPerson(context.Background(), Person{Name: "ada"}); err == nil { + t.Fatal("somebody who may call nothing was recorded") + } +} + +// Forgetting somebody takes their credential with them. **Both, or it is not a revocation**: a +// person's row gone and their bus user left behind is a credential that still works and that nothing +// derives. +func TestForgettingAPersonTakesTheirCredential(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + if err := inv.RecordPerson(ctx, Person{Name: "ada", Invokes: []string{"a.one"}}); err != nil { + t.Fatal(err) + } + if _, err := inv.MintBusPassword(ctx, BusUser{Username: "person.ada", Kind: BusPerson}); err != nil { + t.Fatal(err) + } + + if err := inv.ForgetPerson(ctx, "ada"); err != nil { + t.Fatal(err) + } + if _, known, err := inv.BusUserHash(ctx, "person.ada"); err != nil || known { + t.Fatalf("a forgotten person's credential still works: %v %v", known, err) + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := records.People["ada"]; still { + t.Fatal("a forgotten person is still composed into the bus") + } +} diff --git a/internal/link/build.go b/internal/link/build.go index 2ebe602..f097233 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -82,6 +82,16 @@ type BuildResult struct { // about the source. On string `json:"on"` + // Module is what was built, read out of the manifest — the only place it is authoritative, since + // a request names a repository and a path. + // + // **Here because one message now reaches three audiences** (novox/hq ADR 0121). On the bus the + // mesh runs on today the answer and the announcement were two publishes to two topologies, so a + // result needed no module name and the announcement carried one. On the bus being built the + // outcome is the role's own event, and the catalogue reading it needs to know what was built. + // Empty on a failed build, which produced no module version. + Module string `json:"module,omitempty"` + // Commit is what was actually built. The mesh records it, which is what makes "is this // current?" answerable without building again. Commit string `json:"commit,omitempty"` diff --git a/internal/link/builds.go b/internal/link/builds.go new file mode 100644 index 0000000..8963bc2 --- /dev/null +++ b/internal/link/builds.go @@ -0,0 +1,108 @@ +package link + +import ( + "context" + "encoding/json" + "fmt" + "time" +) + +// Asking a role to build something, and being told what came of it. +// +// A build is work submitted to a role, not a message to a machine (novox/hq ADR 0121). The +// build-machine seat accepts a build and emits an outcome, so the same publish that answers whoever +// asked also reaches the controller that records it and the catalogue that places it in the module +// graph — and no build machine needs permission to publish into anybody's inbox. +// +// **This is the one flow whose shape differs from every other**, which is why it has its own seam +// rather than living in `Bus`. Everything else the controller sends is either an event nobody must +// act on or a declaration a node reconciles toward; a build is a request that takes minutes and has +// exactly one answer. Too long for request/reply, too particular to be an event. + +// TheBuildMachine is the role a build is submitted to. +const TheBuildMachine = "mesh-build-machine" + +// BuildWork is where a build request lands, and BuildOutcome is where its result does. Derived from +// the seat, so both sides name the role and neither names the other. +func BuildWork() string { return "mesh.seat." + TheBuildMachine + ".accept.build" } +func BuildOutcome() string { return "mesh.seat." + TheBuildMachine + ".event.built" } + +// KeyRoleBuilt is the build outcome under the role's name, on the bus the mesh runs on today. +// +// The same event as KeyModuleBuilt and published beside it, because a catalogue installed before this +// change listens for the module's name and one installed after listens for the role's. Both, until +// this bus retires: a rename needs publisher and subscriber to change together, and a deployment +// cannot promise which arrives first. +const KeyRoleBuilt = "built" + +// Builders is how work reaches a build machine and how the outcome comes back. +type Builders interface { + // Submit asks for one build and waits for its outcome. + // + // The wait is long by nature. A build clones, pulls a base image and runs a container build, so + // a timeout here says "nothing is doing builds" rather than "this build is slow" — and the two + // need different remedies, which is why the message distinguishes them. + Submit(ctx context.Context, request BuildRequest, wait time.Duration) (BuildResult, error) + + // Close lets go of whatever was dialled. + Close() +} + +// BuildMachine is a machine taking work from the role it holds. +type BuildMachine interface { + // Take hands each request to do until the context ends, and says why it stopped. + Take(ctx context.Context, do func(context.Context, Build)) error + Close() +} + +// Build is one request a machine has been handed. +type Build interface { + // Request is what to build. + Request() BuildRequest + + // Announce publishes the outcome as the role's own event. + // + // One publish, three audiences: whoever asked matches it by the id their request carried, the + // controller records it, and the catalogue places it. On the bus the mesh runs on today that + // fan-out came from a shared exchange; here the mesh derived the subject. + Announce(ctx context.Context, result BuildResult) error + + // Done settles the request. Called only after the outcome is away, so a machine that dies + // before announcing leaves the work for another rather than losing it. + Done() error + + // Hold hands the work back for another attempt after the delay. + Hold(after time.Duration) error +} + +// waitingFor is the message a caller gets when nothing answered. Its own function because both +// transports say it, and saying it differently in two places is how one of them ends up vague. +func waitingFor(wait time.Duration) error { + return fmt.Errorf( + "no build machine answered within %s. Either nothing holds %s — in which case the work is "+ + "queued and will be done when something does — or a build is taking longer than this", + wait, TheBuildMachine) +} + +// theOutcomeOf reads a result and says whether it is the answer to this request. +func theOutcomeOf(body []byte, id string) (BuildResult, bool, error) { + var result BuildResult + if err := json.Unmarshal(body, &result); err != nil { + return BuildResult{}, false, fmt.Errorf("a build machine answered with something unreadable: %w", err) + } + // Somebody else's build. Skipped rather than returned, because returning it would attribute one + // build's outcome to another's. + return result, result.ID == id, nil +} + +// ModuleOf reads the module's name out of a manifest a build produced, which is the only place it is +// authoritative — a request named a repository and a path, not a module. +func ModuleOf(manifest json.RawMessage) string { + var named struct { + Module string `json:"module"` + } + if err := json.Unmarshal(manifest, &named); err != nil { + return "" + } + return named.Module +} diff --git a/internal/link/builds_current.go b/internal/link/builds_current.go new file mode 100644 index 0000000..57165d2 --- /dev/null +++ b/internal/link/builds_current.go @@ -0,0 +1,202 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The build flow on the bus the mesh runs on today. +// +// Moved behind the seam rather than changed. The queue, the reply binding and the correlation are +// what they were, because the mesh is running on this. + +// currentBuilds asks for builds over a channel. +type currentBuilds struct{ channel *amqp.Channel } + +// BuildsOverCurrent is the asking side on the bus the mesh has. +func BuildsOverCurrent(channel *amqp.Channel) Builders { return currentBuilds{channel: channel} } + +func (b currentBuilds) Close() {} + +func (b currentBuilds) Submit(ctx context.Context, request BuildRequest, + wait time.Duration) (BuildResult, error) { + + // Its own queue for the answer, declared before the ask. Consuming from the shared exchange + // would mean competing with the controller's own consumer for a message meant for this caller. + replies, err := b.channel.QueueDeclare(ReplyQueue(request.ID), false, true, true, false, nil) + if err != nil { + return BuildResult{}, err + } + // **A builder never publishes to the default exchange**, because permission there is per + // exchange and not per queue — a builder allowed to use it could publish into any node's queue, + // which is the privilege a build machine most obviously should not have. The cost is that every + // asker sees every result, which is why the correlation is checked below rather than assumed. + if err := b.channel.QueueBind(replies.Name, KeyBuilt, Exchange, false, nil); err != nil { + return BuildResult{}, err + } + answers, err := b.channel.ConsumeWithContext(ctx, replies.Name, "", true, true, false, false, nil) + if err != nil { + return BuildResult{}, err + } + + body, err := json.Marshal(request) + if err != nil { + return BuildResult{}, err + } + if err := b.channel.PublishWithContext(ctx, "", BuildQueue, false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + CorrelationId: request.ID, + ReplyTo: replies.Name, + Body: body, + }); err != nil { + return BuildResult{}, err + } + + waiting, cancel := context.WithTimeout(ctx, wait) + defer cancel() + for { + select { + case <-waiting.Done(): + return BuildResult{}, waitingFor(wait) + case delivery, ok := <-answers: + if !ok { + return BuildResult{}, errors.New("the connection closed while waiting for a build") + } + result, mine, err := theOutcomeOf(delivery.Body, request.ID) + if err != nil { + return BuildResult{}, err + } + if mine { + return result, nil + } + } + } +} + +// --- the machine's side --------------------------------------------------------------------- + +type currentMachine struct { + conn *amqp.Connection + channel *amqp.Channel + on string +} + +// MachineOverCurrent takes build work over a channel. +func MachineOverCurrent(conn *amqp.Connection, channel *amqp.Channel, on string) BuildMachine { + return ¤tMachine{conn: conn, channel: channel, on: on} +} + +func (m *currentMachine) Close() {} + +func (m *currentMachine) Take(ctx context.Context, do func(context.Context, Build)) error { + if _, err := m.channel.QueueDeclare(BuildQueue, true, false, false, false, nil); err != nil { + return err + } + // One at a time. A machine that took five requests at once would run five container builds + // against one runtime and finish all of them slower than it would have finished the first — and + // the queue is what shares work between machines, so nothing is lost by it. + if err := m.channel.Qos(1, 0, false); err != nil { + return err + } + // Not auto-acknowledged: a request acknowledged on arrival is a build that vanishes if this + // process dies mid-way, with nobody waiting on it ever hearing why. + requests, err := m.channel.ConsumeWithContext(ctx, BuildQueue, "mesh-builder", + false, false, false, false, nil) + if err != nil { + return err + } + for { + select { + case <-ctx.Done(): + return nil + case delivery, ok := <-requests: + if !ok { + return errors.New("the broker closed the connection") + } + var request BuildRequest + if err := json.Unmarshal(delivery.Body, &request); err != nil { + // Unreadable: rejected rather than retried, because the next attempt reads the same + // bytes. Nobody waiting hears an answer, which is correct — there was no request. + _ = delivery.Reject(false) + continue + } + do(ctx, ¤tBuild{request: request, delivery: delivery, on: m.on, channel: m.channel}) + } + } +} + +type currentBuild struct { + request BuildRequest + delivery amqp.Delivery + on string + channel *amqp.Channel +} + +func (b *currentBuild) Request() BuildRequest { return b.request } + +// Announce answers and announces, which on this bus are two publishes to two exchanges. +// +// The reply goes to whoever asked, correlated to their request; the announcement says to the whole +// mesh that a module now exists at a commit (novox/hq ADR 0072). Only a successful build is +// announced: a failed one produced no module version, and announcing one would put something in the +// graph that was never made. +func (b *currentBuild) Announce(ctx context.Context, result BuildResult) error { + body, err := json.Marshal(result) + if err != nil { + return err + } + if err := b.channel.PublishWithContext(ctx, Exchange, KeyBuilt, false, false, amqp.Publishing{ + ContentType: "application/json", + CorrelationId: result.ID, + Body: body, + }); err != nil { + return fmt.Errorf("cannot answer a build request: %w", err) + } + if result.Failed != "" || result.Commit == "" { + return nil + } + + // **Announced under both names on this bus, for exactly as long as this bus lives.** + // + // A build's outcome belongs to the role now (novox/hq ADR 0121), so a catalogue built from the + // current manifests listens for the role's name. A catalogue that is *already running* listens for + // the module's, because that is what it was told when it was installed. A rename on a live bus + // needs the publisher and the subscriber to change together, and a merge cannot promise that: one + // of them is deployed first, and in that window the graph silently stops being updated — which is + // the failure this whole change was cleaning up after. + // + // So both, and the order stops mattering. The module's own name goes with the bus, in step 5's + // retirement list; nothing has ever run on the bus being built, so there is no legacy name there + // and this doubling has no counterpart. + announced := announcementOf(result) + if err := EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyModuleBuilt, "builder", b.on, + announced); err != nil { + return err + } + return EmitEvent(ctx, OverCurrent{Channel: b.channel}, KeyRoleBuilt, TheBuildMachine, b.on, + announced) +} + +func (b *currentBuild) Done() error { return b.delivery.Ack(false) } + +func (b *currentBuild) Hold(time.Duration) error { + // No delayed redelivery on this bus: handed back at once, which is what it has always done. + return b.delivery.Nack(false, true) +} + +// announcementOf is what the mesh is told about a finished build. One function, so the two +// transports cannot describe the same build differently. +func announcementOf(result BuildResult) map[string]any { + return map[string]any{ + "module": ModuleOf(result.Manifest), "commit": result.Commit, + "repository": result.Repository, "path": result.Path, "ref": result.Ref, + "manifest": json.RawMessage(result.Manifest), "against": result.Against, + "made": result.Made, + } +} diff --git a/internal/link/builds_current_test.go b/internal/link/builds_current_test.go new file mode 100644 index 0000000..7f43f2c --- /dev/null +++ b/internal/link/builds_current_test.go @@ -0,0 +1,36 @@ +package link + +import ( + "strings" + "testing" +) + +// **The old bus announces a build under both names, and that is not belt-and-braces.** +// +// A build's outcome belongs to the role now, so a catalogue built from the current manifests listens +// for the role's name — and a catalogue already running listens for the module's, because that is what +// it was told when it was installed. A rename on a live bus needs publisher and subscriber to change +// together, which a deployment cannot promise: one arrives first, and in that window the module graph +// silently stops being updated. +// +// Caught by asking what merging this would do to the mesh that is actually running, which is the only +// place the question could have been asked — the tests were green and both buses were self-consistent. +func TestTheOldBusAnnouncesABuildUnderBothNames(t *testing.T) { + // The routing key a catalogue installed before the change is bound to. + if KeyModuleBuilt != "module.builder.built" { + t.Fatalf("the module's own name is %q; a catalogue already running is bound to the old one", + KeyModuleBuilt) + } + // And the local name a catalogue built from the current manifests declares, which the old bus's + // client turns into `module..built`. + if KeyRoleBuilt != "built" { + t.Fatalf("the role's event is %q, and a holder emits its verbs bare", KeyRoleBuilt) + } + if TheBuildMachine != "mesh-build-machine" { + t.Fatalf("the role is %q", TheBuildMachine) + } + // The two must differ, or one publish would serve both and this doubling would be pointless. + if strings.HasSuffix(KeyModuleBuilt, "."+TheBuildMachine+"."+KeyRoleBuilt) { + t.Fatal("the two names are the same, so nothing was renamed and this is dead weight") + } +} diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go new file mode 100644 index 0000000..039d340 --- /dev/null +++ b/internal/link/builds_nats.go @@ -0,0 +1,206 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The build flow on the bus being built. +// +// **One publish where the old bus needed two** (novox/hq ADR 0121). There, the answer went to a +// reply queue and the announcement to an events exchange, because the two audiences were reached by +// two topologies. Here the outcome is the role's own event: whoever asked matches it by the id their +// request carried, the controller records it, the catalogue places it in the graph. So a build +// machine publishes once and needs permission for nothing but its own role's subjects — no reply +// queue to declare, and no grant over anybody's inbox. + +// natsBuilds asks for builds over a connection. +type natsBuilds struct { + js *broker.JetStream + owned bool +} + +// BuildsOverNATS is the asking side on the bus being built. It dials, because the command that asks +// for a build is a one-shot and holds nothing else. +func BuildsOverNATS(address string) (Builders, error) { + js, err := broker.Dial(address) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus at %s to ask for a build: %w", address, err) + } + return &natsBuilds{js: js, owned: true}, nil +} + +func (b *natsBuilds) Close() { + if b.owned && b.js != nil { + b.js.Close() + } +} + +func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest, + wait time.Duration) (BuildResult, error) { + + // Subscribed before the ask, so an outcome cannot arrive before there is anywhere for it to + // land. Core, not the stream: the asker is waiting now, and the durable copy of this outcome is + // the same event on EVENTS, which the controller records. + outcomes, err := b.js.Conn().SubscribeSync(BuildOutcome()) + if err != nil { + return BuildResult{}, fmt.Errorf("cannot listen for a build's outcome: %w", err) + } + defer func() { _ = outcomes.Unsubscribe() }() + if err := b.js.Conn().Flush(); err != nil { + return BuildResult{}, err + } + + body, err := json.Marshal(request) + if err != nil { + return BuildResult{}, err + } + // Into the role's work queue and awaited: work the bus never accepted must fail here rather than + // be assumed, because nothing else will ever say so. + publish, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if _, err := b.js.Context().Publish(BuildWork(), body, nats.Context(publish)); err != nil { + return BuildResult{}, fmt.Errorf("cannot submit a build: %w", err) + } + + waiting, cancelWait := context.WithTimeout(ctx, wait) + defer cancelWait() + for { + msg, err := outcomes.NextMsgWithContext(waiting) + switch { + case errors.Is(err, context.DeadlineExceeded): + return BuildResult{}, waitingFor(wait) + case errors.Is(err, context.Canceled): + return BuildResult{}, ctx.Err() + case err != nil: + return BuildResult{}, fmt.Errorf("waiting for a build's outcome: %w", err) + } + result, mine, err := theOutcomeOf(msg.Data, request.ID) + if err != nil { + return BuildResult{}, err + } + if mine { + return result, nil + } + } +} + +// --- the machine's side --------------------------------------------------------------------- + +type natsMachine struct { + js *broker.JetStream + on string + seat string + sub *nats.Subscription +} + +// MachineOverNATS takes build work from the role this machine holds. +func MachineOverNATS(js *broker.JetStream, on string) BuildMachine { + return &natsMachine{js: js, on: on, seat: TheBuildMachine} +} + +func (m *natsMachine) Close() { + if m.sub != nil { + _ = m.sub.Unsubscribe() + } +} + +// Take binds to the role's worker and hands each request over, one at a time. +// +// **Bound, never created.** The work queue and the worker on it are the controller's to define +// (design 25 §3), and a build machine reaches no part of the JetStream API — so a missing one is said +// as the mesh's to answer rather than quietly created with whatever this client defaults to. +func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error { + worker, found := broker.HolderConsumerFor(m.on, "builder", + broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}}) + if !found { + return fmt.Errorf("%s accepts no work, so there is nothing for this machine to take", m.seat) + } + + // One at a time, which the consumer's own ack-pending limit enforces rather than a prefetch + // setting: a machine that took five requests at once would run five container builds against one + // runtime and finish all of them slower than the first. + work := make(chan *nats.Msg, 1) + // **The consumer's own filter, not the one subject this machine cares about.** The client checks + // what is asked for against the consumer's filter and refuses anything that is not the same — + // "subject does not match consumer" — so subscribing `…accept.build` against a consumer filtered + // on `…accept.>` is rejected even though it is narrower. Learned twice now, on two different + // consumers, which is why it is written down here. + filter := worker.Filters[0] + sub, err := m.js.Context().ChanQueueSubscribe(filter, worker.Queue, work, + nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + return fmt.Errorf( + "this machine cannot take work from %s: %w. The mesh creates that queue and this "+ + "machine's worker on it, and a build machine may not create one itself — so this is "+ + "the mesh's to answer, not this machine's", m.seat, err) + } + m.sub = sub + + for { + select { + case <-ctx.Done(): + return nil + case msg, ok := <-work: + if !ok { + return errors.New("the bus stopped delivering build work") + } + var request BuildRequest + if err := json.Unmarshal(msg.Data, &request); err != nil { + // Unreadable: terminated rather than retried, because the next attempt reads the same + // bytes. Nobody waiting hears an answer, which is right — there was no request. + _ = msg.Term() + continue + } + do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js}) + } + } +} + +type natsBuild struct { + request BuildRequest + msg *nats.Msg + on string + js *broker.JetStream +} + +func (b *natsBuild) Request() BuildRequest { return b.request } + +// Announce publishes the outcome as the role's own event, once, for all three audiences. +// +// Into the stream, so a controller that was restarting still records it and a catalogue that was +// down still catches up. The asker is listening on core for the same subject and gets it either way: +// a stream delivers to its durable consumers and the plain subscribers both. +func (b *natsBuild) Announce(ctx context.Context, result BuildResult) error { + // One body, three readers. Whoever asked matches the id; the controller records it; the catalogue + // needs to know what was built, which only the manifest says — so the result carries it rather + // than a second message carrying a second shape. + // + // **A failed build names no module**, because it produced no module version and the catalogue + // would otherwise put something in the graph that was never made. The asker still gets its + // answer: a failure is the answer. + if result.Failed == "" && result.Commit != "" { + result.Module = ModuleOf(result.Manifest) + } + body, err := json.Marshal(result) + if err != nil { + return err + } + publish, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if _, err := b.js.Context().Publish(BuildOutcome(), body, nats.Context(publish)); err != nil { + return fmt.Errorf("cannot announce a build's outcome: %w", err) + } + return nil +} + +func (b *natsBuild) Done() error { return b.msg.Ack() } + +func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) } diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go new file mode 100644 index 0000000..2672258 --- /dev/null +++ b/internal/link/builds_nats_test.go @@ -0,0 +1,215 @@ +package link + +import ( + "context" + "encoding/json" + "io" + "log" + "os" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// A build, end to end, against a real server. +// +// The claim worth checking is the one ADR 0121 rests on: **one publish reaches three audiences**. +// Whoever asked matches the outcome by the id their request carried; the controller records it; the +// catalogue places it. On the old bus that fan-out came from a shared exchange, and it would be easy +// to write a version where only the asker hears it and nobody notices for weeks. +// +// docker run -d --rm --name t -p 14230:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14230 go test ./internal/link/ -run TestNatsABuild + +func aBusWithTheBuildRole(t *testing.T) *broker.JetStream { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + + seats := []broker.DeclaredSeat{{Name: TheBuildMachine, Accepts: []string{"build"}, + Emits: []string{"built"}}} + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := broker.RaiseSeats(js, seats, map[string]broker.Holder{ + TheBuildMachine: {Node: "anchor", Module: "builder"}, + }); err != nil { + t.Fatal(err) + } + clean := func() { + _ = js.Context().DeleteStream("SEAT_MESH_BUILD_MACHINE") + for _, s := range broker.MeshStreams() { + _ = js.Context().PurgeStream(s.Name) + } + } + t.Cleanup(clean) + return js +} + +// The whole round trip: asked, taken, built, and the outcome heard by the asker and by a consumer of +// the role's event who never asked for anything. +func TestNatsABuildIsTakenAndItsOutcomeReachesEverybody(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + // A third party on the role's event — what the catalogue is. Subscribed first, so nothing is + // missed. + heard := make(chan BuildResult, 4) + watching, err := js.Conn().Subscribe(BuildOutcome(), func(msg *nats.Msg) { + var r BuildResult + if json.Unmarshal(msg.Data, &r) == nil { + heard <- r + } + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = watching.Unsubscribe() }() + _ = js.Conn().Flush() + + // A build machine holding the role. + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + failed := make(chan error, 1) + go func() { + failed <- machine.Take(ctx, func(ctx context.Context, work Build) { + r := work.Request() + _ = work.Announce(ctx, BuildResult{ + ID: r.ID, Repository: r.Repository, On: "anchor", Commit: "abc1234", + Manifest: json.RawMessage(`{"module":"shop"}`), + }) + _ = work.Done() + }) + }() + + ask := &natsBuilds{js: js} + result, err := ask.Submit(ctx, BuildRequest{ID: "b-1", Repository: "/r"}, 15*time.Second) + if err != nil { + select { + case why := <-failed: + t.Fatalf("the machine could not take work: %v", why) + default: + } + t.Fatalf("the asker never got an outcome: %v", err) + } + if result.ID != "b-1" || result.Commit != "abc1234" { + t.Fatalf("the asker got %+v", result) + } + // Named in the outcome, because only the manifest says what was built and the catalogue reading + // this event needs to know. + if result.Module != "shop" { + t.Errorf("the outcome names module %q, so a catalogue reading it cannot place the build", + result.Module) + } + + select { + case also := <-heard: + if also.ID != "b-1" { + t.Fatalf("a third party heard %+v", also) + } + case <-time.After(5 * time.Second): + t.Fatal("nobody but the asker heard the outcome, so the catalogue would never place the build") + } + + // And the work left the queue: a request a machine took and settled must not be given to another. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE") + if err == nil && info.State.Msgs == 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the request is still queued after being settled, so another machine would build it again") +} + +// Work submitted with no machine holding the role waits rather than failing, and is done when one +// arrives. **That is what a queue is for**, and the alternative — refusing because nobody is there +// yet — would make installing a build machine an ordering problem. +func TestNatsABuildWaitsForAMachineRatherThanFailing(t *testing.T) { + js := aBusWithTheBuildRole(t) + + body, _ := json.Marshal(BuildRequest{ID: "b-2", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + info, err := js.Context().StreamInfo("SEAT_MESH_BUILD_MACHINE") + if err != nil || info.State.Msgs != 1 { + t.Fatalf("the work did not queue: %+v %v", info, err) + } + + // Now a machine arrives and finds it waiting. + ctx, stop := context.WithCancel(context.Background()) + defer stop() + took := make(chan string, 1) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-took: + if id != "b-2" { + t.Fatalf("the machine took %q", id) + } + case <-time.After(10 * time.Second): + t.Fatal("a machine that arrived after the work did never got it, so the backlog was lost") + } +} + +// A machine that dies before saying anything leaves the work for another. +func TestNatsWorkAMachineDidNotAnswerGoesBackToTheQueue(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + body, _ := json.Marshal(BuildRequest{ID: "b-3", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + + var once sync.Once + handed := make(chan struct{}, 2) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + handed <- struct{}{} + // The first time, hand it straight back — a machine that stopped mid-build. + var settled bool + once.Do(func() { _ = work.Hold(200 * time.Millisecond); settled = true }) + if !settled { + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor"}) + _ = work.Done() + } + }) + }() + + for i := 0; i < 2; i++ { + select { + case <-handed: + case <-time.After(10 * time.Second): + t.Fatalf("the work was handed over %d time(s); unanswered work must come back", i) + } + } +} + +func quietLog() *log.Logger { return log.New(io.Discard, "", 0) } + +var _ = quietLog diff --git a/internal/link/bus.go b/internal/link/bus.go new file mode 100644 index 0000000..b96d3cf --- /dev/null +++ b/internal/link/bus.go @@ -0,0 +1,194 @@ +package link + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go" + amqp "github.com/rabbitmq/amqp091-go" +) + +// Bus is what the controller needs of the mesh's bus, **in the mesh's own words rather than a +// transport's** (novox/hq ADR 0116 step 3). +// +// Until now every one of these functions took the transport's own channel type, so the transport +// reached every caller and changing it meant touching all of them. The seam is small — the +// controller sends exactly two kinds of message that expect no answer, and asks two kinds of +// question — which is why the bus can be replaced at all. +// +// Two implementations live below, and both ship until the rollout (ADR 0116: nothing moves a +// node's bus before step 5). Both shipping is what makes them comparable — the same caller, the +// same arguments, and one conformance fixture holding them to one envelope. +type Bus interface { + // PublishEvent announces something that happened, under the emitter's own name. 1:many, and + // nobody is obliged to act (ADR 0041). + PublishEvent(ctx context.Context, key, source, node string, body []byte) error + + // PublishDeclaration delivers one node what it should be. Addressed to that node alone: a + // declaration is not an event, and replaying yesterday's is actively harmful + // (design 29 §4, the *state* shape). + PublishDeclaration(ctx context.Context, node string, body []byte) error + + // AskTool sends one question to a module's tool and awaits one answer. A tool nobody serves + // must say so **at once** rather than after the whole wait: the difference between "that + // module is down" and "that tool is slow" is the first thing a person asking wants. + AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) +} + +// --- The bus the mesh runs on today ----------------------------------------------------- + +// OverCurrent is the bus the mesh runs on today, until the rollout. +type OverCurrent struct{ Channel *amqp.Channel } + +func (b OverCurrent) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { + id, err := eventID() + if err != nil { + return err + } + return b.Channel.PublishWithContext(ctx, EventsExchange, key, false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + MessageId: id, + Timestamp: time.Now().UTC(), + Body: body, + Headers: amqp.Table{ + "x-event-id": id, + "x-source": source, + "x-node": node, + "x-time": time.Now().UTC().Format(time.RFC3339), + "content-type": "application/json", + }, + }) +} + +// AskTool is implemented over the existing reply-queue machinery in ask.go; this seam does not +// change how it works today. +func (b OverCurrent) AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) { + answer, err := Ask(ctx, b.Channel, module, tool, args, timeout) + if err != nil { + return nil, err + } + return answer.Result, nil +} + +func (b OverCurrent) PublishDeclaration(ctx context.Context, node string, body []byte) error { + // To the queue directly rather than through an exchange: a declaration is for one node, and + // routing it by name through a shared exchange would mean a binding per node that nothing + // removes when a node is retired. + return b.Channel.PublishWithContext(ctx, "", QueueFor(node), false, false, amqp.Publishing{ + ContentType: "application/json", + DeliveryMode: amqp.Persistent, + Body: body, + }) +} + +// --- NATS, the bus being built ---------------------------------------------------------------- + +// OverNATS is the bus as a JetStream context. +type OverNATS struct { + Conn *nats.Conn + JS nats.JetStreamContext +} + +// The subjects a node publishes on, and the controller listens to. +// +// One tree, and each name says who it is about: `mesh.control..…` is a node's own, which is +// what lets a node's account be granted exactly its own prefix and nothing of any other node's +// (design 25 §2, §4). The two that belong to no node — an enrolment, because a machine enrolling +// has no name the mesh has agreed to yet, and a build's outcome, because a builder is not +// reporting about itself — are named directly. +const ( + // EnrolSubject is where a joining machine asks. Its enrolment user may publish here and + // nowhere else, so a leaked token buys nothing but the chance to enrol. + EnrolSubject = "mesh.control.enrol" + + // BuiltSubject is where a build's outcome lands, for results nobody was waiting for. + BuiltSubject = "mesh.control.built" + + // AliveSubjects is every node's heartbeat. Core NATS, never a stream: a lost heartbeat is the + // next heartbeat, and a stream of them is the mesh's least valuable message competing for + // retention with its most valuable (design 25 §3). + AliveSubjects = "mesh.control.*.alive" +) + +// ReportSubject is where one node says what it did. On the CONTROL stream, because it is the +// message the store-window guarantee is about (ADR 0083). +func ReportSubject(node string) string { return "mesh.control." + node + ".report" } + +// AliveSubject is one node's heartbeat. +func AliveSubject(node string) string { return "mesh.control." + node + ".alive" } + +// EventSubject is where a module's event lands. Derived from the emitter, never taken from the +// caller: a source that could differ from the subject is an envelope that can lie about its +// origin, and on NATS the account's permissions make the subject the authority (design 29 §2). +func EventSubject(source, key string) string { + return "mesh.mod." + source + ".event." + key +} + +// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so +// a node that was away gets exactly the current one and a replayed older one is refused by +// sequence — the wire-level answer to novox/hq issue 107. +func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" } + +func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { + id, err := eventID() + if err != nil { + return err + } + h := nats.Header{} + h.Set("x-event-id", id) + h.Set("x-source", source) + h.Set("x-node", node) + h.Set("x-time", time.Now().UTC().Format(time.RFC3339)) + h.Set("content-type", "application/json") + + // The id is also the publish's message id, so the server refuses a duplicate inside its + // window. That narrows the window a consumer must deduplicate in; it does not remove the + // requirement, because the window is finite (design 19, delivery). + _, err = b.JS.PublishMsg(&nats.Msg{ + Subject: EventSubject(source, key), + Header: h, + Data: body, + }, nats.MsgId(id), nats.Context(ctx)) + if err != nil { + return fmt.Errorf("emitting %s: %w", key, err) + } + return nil +} + +func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error { + _, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx)) + if err != nil { + return fmt.Errorf("declaring to %s: %w", node, err) + } + return nil +} + +// ToolSubject is where a module answers. Derived from the module and the tool, so a caller names +// what it wants rather than where it lives. +func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool } + +func (b OverNATS) AskTool(ctx context.Context, module, tool string, args []byte, timeout time.Duration) ([]byte, error) { + if len(args) == 0 { + args = []byte(`{}`) + } + ask, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // **No reply queue, and no correlation to check.** The caller's inbox is its own — each + // account is granted one prefix and no other (design 25 §4) — so an answer cannot reach the + // wrong asker and there is nothing to correlate against. That also settles a cost recorded + // in build.go: on a shared reply exchange every asker saw every result. + msg, err := b.Conn.RequestWithContext(ask, ToolSubject(module, tool), args) + if err != nil { + if errors.Is(err, nats.ErrNoResponders) { + // Said at once rather than after the whole wait: nothing is subscribed to that + // subject, which is a different fact from a tool being slow. + return nil, fmt.Errorf("nothing serves %s.%s", module, tool) + } + return nil, fmt.Errorf("asking %s.%s: %w", module, tool, err) + } + return msg.Data, nil +} diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go new file mode 100644 index 0000000..909c4db --- /dev/null +++ b/internal/link/bus_test.go @@ -0,0 +1,154 @@ +package link + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// **Both implementations, one fixture.** The point of the seam is not that the transport can be +// swapped — it is that the two can be held to the same envelope while both ship, so the day the +// bus moves is a configuration change rather than a discovery. +// +// Against a real server, because what the fixture pins is what reaches the wire: +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestTheNatsBus +func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + f := loadFixture(t, "events/module-event.json") + + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.AddStream(&nats.StreamConfig{ + Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}, + }); err != nil && err != nats.ErrStreamNameAlreadyInUse { + t.Fatal(err) + } + + bus := OverNATS{JS: js} + body, _ := json.Marshal(f.Given.Body) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil { + t.Fatal(err) + } + + // Read back from the stream, not from the thing that wrote it. + raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject) + if err != nil { + t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err) + } + for _, h := range f.Wire.RequiredHeaders { + if raw.Header.Get(h) == "" { + t.Errorf("%s is not set on the wire, and the fixture requires it", h) + } + } + if got := raw.Header.Get("x-source"); got != f.Given.Module { + t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module) + } + if string(raw.Data) != string(body) { + t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body) + } + // The envelope must not also be nested inside the payload. + var nested map[string]any + if json.Unmarshal(raw.Data, &nested) == nil { + if _, has := nested["key"]; has { + t.Error("the payload carries the envelope's own fields, which the fixture refuses") + } + } +} + +// The subject a declaration lands on is one node's, and nothing else's — the state shape. +func TestADeclarationIsAddressedToOneNode(t *testing.T) { + if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" { + t.Fatalf("a declaration would go to %q", got) + } + if DeclareSubject("anchor") == DeclareSubject("laptop") { + t.Fatal("two nodes share a declaration subject, so each would apply the other's") + } +} + +// A module cannot emit under another's name: the subject is derived from the source, and the +// server's permissions make that subject the authority. +func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { + if got := EventSubject("shop", "order.placed"); got != "mesh.mod.shop.event.order.placed" { + t.Fatalf("an event would land on %q", got) + } + if EventSubject("shop", "x") == EventSubject("billing", "x") { + t.Fatal("two modules share an event subject, so neither owns its own name") + } +} + +// A tool nobody serves says so at once. The difference between "that module is down" and "that +// tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it +// is how a fast answer becomes a slow non-answer. +func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + bus := OverNATS{Conn: conn} + began := time.Now() + _, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second) + if err == nil { + t.Fatal("asking a tool nothing serves succeeded") + } + if took := time.Since(began); took > 2*time.Second { + t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took) + } + if !strings.Contains(err.Error(), "nothing serves") { + t.Errorf("the refusal does not say nobody is there: %v", err) + } +} + +// And a served tool answers, with no reply queue to declare and no correlation to check. +func TestAskingAServedToolAnswers(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) { + _ = m.Respond([]byte(`{"total":12}`)) + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + + got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price", + []byte(`{"qty":4}`), 5*time.Second) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"total":12}` { + t.Fatalf("the answer came back as %s", got) + } +} diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go new file mode 100644 index 0000000..50f60da --- /dev/null +++ b/internal/link/conformance_test.go @@ -0,0 +1,107 @@ +package link + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + "time" +) + +// The Go implementation, held to the shared fixtures (novox/hq ADR 0074, design 19). +// +// **Read from the sdk's conformance directory by sibling path**, the way the lab finds its +// siblings — deliberately not copied here. A fixture copied into each implementation is two +// fixtures, and two fixtures drift, which is the exact failure the suite exists to prevent. +type fixture struct { + Name string `json:"name"` + Given struct { + Module string `json:"module"` + Node string `json:"node"` + Key string `json:"key"` + Body map[string]any `json:"body"` + Headers map[string]string `json:"headers"` + } `json:"given"` + Wire struct { + Subject string `json:"subject"` + RequiredHeaders []string `json:"requiredHeaders"` + HeaderFormats map[string]string `json:"headerFormats"` + } `json:"wire"` +} + +func loadFixture(t *testing.T, name string) fixture { + t.Helper() + path := filepath.Join("..", "..", "..", "mesh-sdk", "conformance", name) + raw, err := os.ReadFile(path) + if err != nil { + t.Skipf("the sdk's conformance fixtures are not beside this checkout: %v", err) + } + var f fixture + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatalf("%s: %v", name, err) + } + return f +} + +// Every header the fixture requires is one this implementation actually sets. +func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + for _, want := range f.Wire.RequiredHeaders { + if _, ok := sent[want]; !ok { + t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+ + "emits is one a conforming consumer refuses", want) + } + } +} + +// And each value is in the shape the fixture pins, because a header present but differently +// formatted is the disagreement that does not announce itself. +func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + + if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] { + t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"]) + } + if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil { + t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err) + } + if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" { + if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) { + t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern) + } + } + // The origin the envelope claims is the one the bus enforces by namespace. A disagreement + // here means the envelope is lying about where it came from. + if sent["x-source"] != f.Given.Module { + t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module) + } +} + +// The subject a module's event lands on is derived, not carried — so this implementation must +// derive the same one the fixture names. +func TestTheGoSubjectMatchesTheFixture(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key + if got != f.Wire.Subject { + t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject) + } +} + +// goEventHeaders is the header set EmitEvent produces, factored so conformance can see it +// without a broker. Kept beside the emitter so the two cannot drift apart silently. +func goEventHeaders(eventType, source, node string) map[string]string { + id, err := eventID() + if err != nil { + panic(err) + } + return map[string]string{ + "x-event-id": id, + "x-source": source, + "x-node": node, + "x-time": time.Now().UTC().Format(time.RFC3339), + "content-type": "application/json", + } +} diff --git a/internal/link/declare.go b/internal/link/declare.go index 9f6fd7e..665c810 100644 --- a/internal/link/declare.go +++ b/internal/link/declare.go @@ -5,8 +5,6 @@ import ( "encoding/json" "fmt" "time" - - amqp "github.com/rabbitmq/amqp091-go" ) // Signer is whatever holds the control plane's signing key. @@ -21,7 +19,7 @@ type Signer interface { // something else, and the node would refuse a declaration that was genuinely the mesh's. // // Published to the node's own queue, which its account alone may read. -func Declare(ctx context.Context, channel *amqp.Channel, signer Signer, node string, +func Declare(ctx context.Context, bus Bus, signer Signer, node string, declaration []byte, timeout time.Duration) error { if !json.Valid(declaration) { @@ -41,13 +39,5 @@ func Declare(ctx context.Context, channel *amqp.Channel, signer Signer, node str publish, cancel := context.WithTimeout(ctx, timeout) defer cancel() - // Published to the queue directly rather than through the exchange: a declaration is for one - // node, and routing it by name through a shared exchange would mean a binding per node that - // nothing removes when a node is retired. - return channel.PublishWithContext(publish, "", QueueFor(node), false, false, - amqp.Publishing{ - ContentType: "application/json", - DeliveryMode: amqp.Persistent, - Body: body, - }) + return bus.PublishDeclaration(publish, node, body) } diff --git a/internal/link/enrol_bus_credential_test.go b/internal/link/enrol_bus_credential_test.go new file mode 100644 index 0000000..9a35404 --- /dev/null +++ b/internal/link/enrol_bus_credential_test.go @@ -0,0 +1,107 @@ +package link_test + +import ( + "crypto/ed25519" + "crypto/rand" + "testing" + "time" + + "golang.org/x/crypto/bcrypt" + + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// What a node is given to come back with, on the bus being built. +// +// **The credential becomes usable at the next composition, not when it is made**, which is the one +// real difference from the bus the mesh runs on today: there a management call makes it live at once. +// So what has to be true here is that the mesh recorded it and told the node, and the rest is a push. + +// aMeshReadyToEnrol is both stores with a signing key established, which a control plane does at +// start: one that cannot sign is one whose declarations every node correctly refuses. +func aMeshReadyToEnrol(t *testing.T) (*inventory.Inventory, *identity.Identity) { + t.Helper() + inv := inventory.ForTest(t) + ident := identity.ForTest(t) + if _, err := ident.Establish(t.Context()); err != nil { + t.Fatal(err) + } + return inv, ident +} + +func aTokenFor(t *testing.T, inv *inventory.Inventory, node string) (string, ed25519.PublicKey) { + t.Helper() + ctx := t.Context() + if _, err := inv.AddNode(ctx, node); err != nil { + t.Fatal(err) + } + issued, err := inv.IssueToken(ctx, node, time.Hour) + if err != nil { + t.Fatal(err) + } + public, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return issued.Secret, public +} + +// A node enrolling onto the bus being built is told a password of its own, and the mesh keeps only +// its hash — which is what the next composition writes into the bus's user list. +func TestANodeEnrollingOnTheNewBusIsMintedACredentialTheMeshOnlyHashes(t *testing.T) { + inv, ident := aMeshReadyToEnrol(t) + ctx := t.Context() + secret, public := aTokenFor(t, inv, "anchor") + + reply, err := link.Enrolment{Inventory: inv, Identity: ident, OnNATS: true}.Enrol(ctx, link.EnrolRequest{ + Node: "anchor", Secret: secret, PublicKey: public}) + if err != nil { + t.Fatal(err) + } + if reply.Password == "" { + t.Fatal("the node was told no password, so it keeps a one-time secret as a credential") + } + if reply.Password == secret { + t.Fatal("the node was handed the token's own secret back: a credential that lives for years " + + "must not be the string that was pasted into a terminal") + } + + // Recorded under the name the composed file will use, and as a hash: a credential recoverable + // from the mesh's store is one whose blast radius is the store's. + hash, known, err := inv.BusUserHash(ctx, "node.anchor") + if err != nil || !known { + t.Fatalf("the mesh kept no credential for the node it enrolled: %v %v", known, err) + } + if hash == reply.Password { + t.Fatal("the store holds the password itself") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(reply.Password)); err != nil { + t.Fatalf("what the mesh kept does not verify what it told the node: %v", err) + } +} + +// On the bus the mesh runs on today, with no management configured, nothing is minted and the node is +// told so by being given no password — it keeps the token's secret, which it says out loud. +// +// **This is the check that the switch is a switch.** A node enrolling on one bus must not come away +// with a credential for the other: it would be half-moved, and nothing anywhere would say which half. +func TestANodeEnrollingOnTheOldBusIsMintedNoCredentialForTheNewOne(t *testing.T) { + inv, ident := aMeshReadyToEnrol(t) + ctx := t.Context() + secret, public := aTokenFor(t, inv, "anchor") + + reply, err := link.Enrolment{Inventory: inv, Identity: ident}.Enrol(ctx, link.EnrolRequest{ + Node: "anchor", Secret: secret, PublicKey: public}) + if err != nil { + t.Fatal(err) + } + if reply.Password != "" { + t.Fatalf("a node on the old bus was given a password from nowhere: %q", reply.Password) + } + if _, known, err := inv.BusUserHash(ctx, "node.anchor"); err != nil || known { + t.Fatalf("a node enrolling on the old bus was given a credential for the new one: %v %v", + known, err) + } +} diff --git a/internal/link/enrol_reply_probe_test.go b/internal/link/enrol_reply_probe_test.go new file mode 100644 index 0000000..f00a879 --- /dev/null +++ b/internal/link/enrol_reply_probe_test.go @@ -0,0 +1,81 @@ +package link + +import ( + "os" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// **Verified 2026-09-27**: the caller asked for a reply to `_INBOX.LCr3M83q…` and the consumer +// saw `$JS.ACK.PROBE.probe_consumer.1.1.1…`. The design was right, and enrolment's payload-borne +// reply subject is necessary rather than defensive. +// +// Design 25 §2 asserts that a reply address is **eaten** when a message crosses a stream: core +// request/reply puts the requester's inbox in the message's Reply field, but a JetStream consumer +// has already claimed that field for its own ack address by the time a handler sees it. The whole +// enrolment design rests on it — the reply subject travels in the payload instead — so it is +// checked rather than believed. +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestAReplyAddress +func TestAReplyAddressDoesNotSurviveAStream(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.AddStream(&nats.StreamConfig{ + Name: "PROBE", Subjects: []string{"probe.>"}, Retention: nats.WorkQueuePolicy, + }); err != nil && err != nats.ErrStreamNameAlreadyInUse { + t.Fatal(err) + } + defer js.DeleteStream("PROBE") + + seen := make(chan *nats.Msg, 1) + sub, err := js.Subscribe("probe.enrol", func(m *nats.Msg) { seen <- m }, + nats.Durable("probe_consumer"), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + + // A caller doing what core request/reply does: publish with its own inbox as the reply. + inbox := nats.NewInbox() + if err := conn.PublishMsg(&nats.Msg{Subject: "probe.enrol", Reply: inbox, Data: []byte("{}")}); err != nil { + t.Fatal(err) + } + + select { + case m := <-seen: + t.Logf("the caller asked for a reply to %s", inbox) + t.Logf("the consumer sees a Reply field of %s", m.Reply) + if m.Reply == inbox { + t.Fatalf("the reply address SURVIVED the stream. Design 25 §2 says it does not, and " + + "builds enrolment around carrying the reply subject in the payload to work " + + "around it. If this holds generally, that work is unnecessary and the design " + + "should say so.") + } + if m.Reply == "" { + t.Fatal("the Reply field is empty rather than claimed; the design says it carries " + + "the consumer's ack address, which is a different fact") + } + // Answering it would send the enrolling node's credentials to an ack subject. + if len(m.Reply) < 7 || m.Reply[:7] != "$JS.ACK" { + t.Errorf("the Reply field is %q, which is neither the caller's inbox nor an ack "+ + "address — the design's reasoning assumes one of the two", m.Reply) + } + m.Ack() + case <-time.After(5 * time.Second): + t.Fatal("nothing was delivered") + } +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 6789377..750ecab 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -28,6 +28,15 @@ type Enrolment struct { Identity *identity.Identity Management *broker.Management Broker broker.Broker + + // OnNATS says the mesh's own traffic is on the bus being built, so a node's credential is + // minted into the mesh's records and composed into the bus's user list rather than pushed + // through a management call (novox/hq design 25 §4). + // + // **One bus, and a node gets a credential for exactly one** — refused at start if the + // controller is told about both (broker.MustBeOneBus), because a node holding a credential for + // each is one that could be half-moved, and nothing would say which half. + OnNATS bool } // Enrol records what the node presented and spends the token. @@ -159,7 +168,33 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol // the spend and not before: a replaced password on an attempt that failed would be held by // nobody. If the broker will not take it now, the enrolment still stands — the node keeps // the token's secret as its password, which it is told, and which is said here. - if e.Management != nil { + switch { + case e.OnNATS: + // **Minted into the mesh's records, not pushed to a server.** The bus's users are a file + // the controller composes, so a credential becomes usable at the next composition rather + // than at the moment it is made — and the plaintext is returned once, here, and then exists + // only on the machine it was sealed to. + // + // The node reconnects as itself and may be refused until that composition reaches the + // machine running the bus. That is what the host's reconnect backoff is for and it is + // survivable by design (ADR 0004: disconnection is an ordinary situation); waiting for the + // push here would hold an enrolment open for as long as a declaration takes to apply. + password, err := e.Inventory.MintBusPassword(ctx, inventory.BusUser{ + Username: broker.Principal{Kind: broker.KindNode, Node: node.Name}.Username(), + Kind: inventory.BusNode, + Node: node.Name, + }) + if err != nil { + // Not fatal to the enrolment: the node is recorded and the token is spent, and a node + // that keeps the token's secret is told so. Said loudly, because until this is minted + // the machine has no credential of its own. + log.Printf("%s is enrolled and the mesh could not mint its bus credential, so it keeps "+ + "the token's secret as its password: %v", node.Name, err) + } else { + reply.Password = password + } + + case e.Management != nil: password, err := freshPassword() if err != nil { return EnrolReply{}, err @@ -234,6 +269,17 @@ var ErrNoBrokerManagement = errors.New("no broker management configured") // A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is // its own account of its own machine, kept as a copy for recovery — so this writes it down and // decides nothing from it. +// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not +// acted on (design 25 §3, window.go). +// +// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great +// many things that record reports have no idea what was sent — every test in this package among +// them. So the serving loop asks for this when the listener happens to be able to answer, and +// where it cannot, a report has nothing to be stale against and is simply acted on. +func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) { + return e.Inventory.Outstanding(ctx, node) +} + func (e Enrolment) Heard(ctx context.Context, report Report) (err error) { // A store that could not be asked right now is said as such, so the report is kept for // another attempt rather than acknowledged and lost (novox/hq issue 082). diff --git a/internal/link/events.go b/internal/link/events.go index 3d461a3..225a19a 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -6,9 +6,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "time" - - amqp "github.com/rabbitmq/amqp091-go" ) // Emitting a module event from Go. @@ -29,32 +26,17 @@ const ( // EmitEvent publishes one module event, in the envelope the sdk's consumers expect. // -// Persistent, because an event that a broker restart loses is not an announcement. The publish is -// not confirmed here: the caller has already done the work the event describes, and a build that -// succeeded must not be reported as failed because saying so failed. -func EmitEvent(ctx context.Context, channel *amqp.Channel, eventType, source, node string, body any) error { +// The envelope is the transport's to write (bus.go) and this is only what goes in it, which is +// what lets one conformance fixture hold both implementations to the same headers. +func EmitEvent(ctx context.Context, bus Bus, eventType, source, node string, body any) error { payload, err := json.Marshal(body) if err != nil { return fmt.Errorf("cannot serialise a %s event: %w", eventType, err) } - id, err := eventID() - if err != nil { - return err - } - return channel.PublishWithContext(ctx, EventsExchange, eventType, false, false, amqp.Publishing{ - ContentType: "application/json", - DeliveryMode: amqp.Persistent, - MessageId: id, - Timestamp: time.Now().UTC(), - Body: payload, - Headers: amqp.Table{ - "x-event-id": id, - "x-source": source, - "x-node": node, - "x-time": time.Now().UTC().Format(time.RFC3339), - "content-type": "application/json", - }, - }) + // The publish is not confirmed by the caller: it has already done the work the event + // describes, and a build that succeeded must not be reported as failed because saying so + // failed. Each transport decides what "published" means for it. + return bus.PublishEvent(ctx, eventType, source, node, payload) } // eventID is what a consumer deduplicates on: delivery is at-least-once, so a handler must be able diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 561f3b6..e8aef82 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -80,6 +80,20 @@ type EnrolRequest struct { // it. Nil from a node that found none, which is every converged one. Tunnel *Tunnel `json:"tunnel,omitempty"` + // ReplyTo is where the answer goes, as a field of the request rather than the transport's own + // reply address. + // + // **Because a stream eats the transport's field** (design 25 §2, verified against a running + // server): a message a JetStream consumer delivers has had its reply field claimed for that + // consumer's own ack address, so by the time the controller sees an enrolment, the field names + // where the *controller* must acknowledge, not where the node is waiting. An enrolment is the + // case that matters — a caller waiting on an ephemeral inbox, over a subject the store window + // may legitimately delay by several nak cycles. + // + // Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the + // field means what it has always meant. + ReplyTo string `json:"reply_to,omitempty"` + // Redelivered is set by the control plane, never sent: the broker handed this request over a // second time. Such a request does not finish an enrolment already spent — the first time may // have answered, and the node holds what it was told. diff --git a/internal/link/receive.go b/internal/link/receive.go new file mode 100644 index 0000000..eda491c --- /dev/null +++ b/internal/link/receive.go @@ -0,0 +1,125 @@ +package link + +import ( + "context" + "time" +) + +// The consume side of the bus, in the mesh's own words. +// +// The outbound half went behind `Bus` (bus.go) and the transport stopped reaching its callers. +// This is the other half, and it is the larger one: everything a node or a module says arrives +// here, and until now every handler took the transport's own delivery type — so the serving loop +// could not be moved to another bus without moving enrolment, reports, builds, upgrades and +// catch-up with it in one breath. +// +// **Two implementations, both shipping** (novox/hq ADR 0116: nothing moves a node's bus before +// step 5). Both shipping is what makes them comparable, and it is what lets the store-window +// guarantee (ADR 0083) be stated once — in window.go, pure — rather than twice, once per +// transport, where the two would eventually disagree about the thing that matters most. + +// The kinds of message the controller acts on. +// +// A transport maps its own addressing onto these — a routing key on the bus the mesh has, a +// subject on the one being built — and nothing past this point knows which it was. They are never +// on the wire: the wire is the transport's business, and a kind that travelled would be a third +// name for the same thing. +const ( + KindEnrolment = "enrolment" + KindReport = "report" + KindHeartbeat = "heartbeat" + KindBuilt = "built" + KindModuleMoved = "module-moved" + KindCatchUp = "catch-up" +) + +// Control is one thing a node or a module said, as the controller must act on it. +// +// **Settling is stated as what the mesh means, not as the transport's verbs.** The two buses +// spell them differently — an ack and a reject against a delivery tag, an ack and a term against +// a stream sequence — and the guarantee is the same either way: `Took` is done with, `Drop` is +// understood and not worth another attempt, and `Hold` is the store window, where the message is +// kept and comes back. +// +// A handler that returns without calling any of the three leaves the message unsettled on +// purpose. That is the right answer while shutting down: a cancelled context is not an answer +// about a message, and the bus should hand it to whatever consumes next (novox/hq issue 083). +type Control interface { + // Kind is which of the constants above this is. + Kind() string + + // Body is the message itself — the payload alone, never the envelope. + Body() []byte + + // Redelivered says the bus has handed this message over before. An enrolment cares and + // nothing else does: one already spent is not finished a second time. + Redelivered() bool + + // HeldFor is how long this message has been waiting to be taken. Zero on a first delivery. + // + // **Read from the message rather than remembered by the controller.** On the bus being built + // it is the age of the publish, which a controller that restarted mid-window still reads + // correctly — the whole reason the holding moves into the server. On the bus the mesh has it + // is how long this process has held it, which is the most that transport can say. + HeldFor() time.Duration + + // Answer replies to whoever is waiting on this message; only an enrolment expects one. + // + // Each transport knows where its own answer goes, and they do not agree about it: one carries + // a reply queue in the delivery, and on the other the field that would have carried it has + // been claimed by the consumer's own ack subject, so the address travels in the payload + // (design 25 §2, verified). That difference is exactly what this seam exists to keep out of + // the handler. + Answer(ctx context.Context, body []byte) error + + // Took settles the message: acted on, or understood and needing no action. + Took() error + + // About names what this message is about — a node's report, one module's move, one build's + // outcome — and is said before the store is asked. + // + // A transport that holds messages **in memory** uses it to set aside anything older it is + // holding about the same thing: the older is the past, and letting it come back after the + // newer was acted on would undo the newer. + // + // **This is the one thing holding-in-memory can do that holding-in-the-server cannot**, and + // naming it here rather than hiding it is deliberate. On the bus being built the message + // belongs to the server and comes back whatever happened meanwhile, so this is ignored and the + // digest a report carries answers the same question instead (window.go, design 25 §3). + About(what string) + + // Hold keeps the message and asks for it again after the delay — the store window. + Hold(after time.Duration) error + + // Drop settles the message without acting on it: refused, stale, or given up on. It is not + // delivered again. + Drop() error +} + +// Inbound is where control messages come from. +type Inbound interface { + // Also asks for one more kind to be delivered. + // + // **Nothing is subscribed unless something is listening for it.** A durable queue or a + // durable stream consumer that nobody reads fills quietly, and the first symptom is a bus out + // of disk rather than anything about modules. + Also(kind string) error + + // Receive delivers every message to act until the context ends, and says why it stopped. + Receive(ctx context.Context, act func(context.Context, Control)) error + + // Close lets go of whatever the implementation holds. + Close() +} + +// Outstanding answers which declaration the mesh last sent a node — the digest, not the +// declaration. +// +// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh +// has already moved past is not worth holding a slot in the window that a current message needs. +// It is a separate interface from Listener rather than a method on it, because a controller that +// only publishes needs neither and something that records reports need not also be able to say +// what was sent. +type Outstanding interface { + Outstanding(ctx context.Context, node string) (string, error) +} diff --git a/internal/link/receive_current.go b/internal/link/receive_current.go new file mode 100644 index 0000000..efeeae5 --- /dev/null +++ b/internal/link/receive_current.go @@ -0,0 +1,317 @@ +package link + +import ( + "context" + "errors" + "fmt" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The consume side on the bus the mesh runs on today. +// +// Everything here was the serving loop's until the seam went in: the queues, the binds, the +// prefetch, and the list of messages the store could not take yet. It moved rather than changed — +// the behaviour this transport has is the behaviour it had, because the mesh is running on it and +// a bus nothing speaks yet is no reason to alter the one every node is on (ADR 0116). + +// Prefetch is how many messages the bus hands the controller before it has settled them. +// +// More than one because a message the store could not take is held, unsettled, while the loop +// goes on answering others — an enrolment above all, which a host is waiting on (novox/hq issue +// 083). Bounded, because what is held is also what the bus has not kept on its own disk as +// pending. +const Prefetch = 64 + +// PrefetchHeadroom is how much of the prefetch is never held, so the loop always has messages to +// answer — an enrolment above all — while others wait for the store. +const PrefetchHeadroom = 8 + +// TryAgainAfter is how often the held are looked at. A store comes back in seconds, and a report a +// few seconds late is still current. +const TryAgainAfter = 2 * time.Second + +// currentInbound consumes what nodes say over the bus the mesh has. +type currentInbound struct { + conn *amqp.Connection + channel *amqp.Channel + // upgrades and catchups are bound only when something is listening (Also). + upgrades bool + catchups bool + // held is every message the store could not take, by the bus's own delivery tag. Kept here + // rather than in the serving loop because holding a delivery unacknowledged is this + // transport's way of keeping it, and the other's is to hand it back to the server. + held map[uint64]*holding + // again is how often the held are looked at; zero means TryAgainAfter. Set by tests. + again time.Duration +} + +// holding is one message kept for the store, and when to try it again. +type holding struct { + message *currentControl + due time.Time + about string +} + +// Current is the consume side of the bus the mesh runs on today. +func Current(conn *amqp.Connection, channel *amqp.Channel) Inbound { + return ¤tInbound{conn: conn, channel: channel, held: map[uint64]*holding{}} +} + +// Also binds the queue one more kind arrives on. +// +// The kinds nodes publish all share one queue and are bound at Connect, because a node may +// publish any of them and binding one while forgetting another is a message the bus accepts, finds +// no queue for, and drops — the publisher sees success and the consumer sees nothing. The two that +// are events get their own queue each, and only when something is listening. +func (c *currentInbound) Also(kind string) error { + switch kind { + case KindModuleMoved: + if err := c.bindEvent(UpgradeQueue, KeyModuleUpgraded); err != nil { + return err + } + c.upgrades = true + case KindCatchUp: + if err := c.bindEvent(CatchUpQueue, KeyCatchingUp); err != nil { + return err + } + c.catchups = true + default: + return fmt.Errorf("nothing binds a queue for %s on this bus", kind) + } + return nil +} + +func (c *currentInbound) bindEvent(queue, key string) error { + if _, err := c.channel.QueueDeclare(queue, true, false, false, false, nil); err != nil { + return fmt.Errorf("cannot declare the %s queue: %w", queue, err) + } + if err := c.channel.QueueBind(queue, key, EventsExchange, false, nil); err != nil { + return fmt.Errorf("cannot bind %s to %s/%s: %w", queue, EventsExchange, key, err) + } + return nil +} + +func (c *currentInbound) Close() {} + +// Receive consumes until the context ends. +// +// One consumer per queue, deliberately: with two on one queue the bus would round-robin between +// them and each would receive half of what it expects — a fault this project has already had, +// between a module's daemon and its capability server. +func (c *currentInbound) Receive(ctx context.Context, act func(context.Context, Control)) error { + // A bounded prefetch rather than one. The loop still takes messages one at a time; what the + // prefetch buys is that a message the store could not take can be held while the loop goes on + // to the next, instead of every enrolment waiting behind it (novox/hq issue 083). Anything + // held goes back to the bus if the controller stops, because nothing held is acknowledged. + if err := c.channel.Qos(Prefetch, 0, false); err != nil { + return err + } + + deliveries, err := c.channel.ConsumeWithContext(ctx, ControlQueue, "control-plane", + false, false, false, false, nil) + if err != nil { + return err + } + + // Its own queue and its own consumer for each event, for the reason above: two consumers on + // one queue split its messages between them, and an upgrade or a catch-up request going to + // whichever half was not listening is a gap that looks like a working mesh. + var upgrades, catchups <-chan amqp.Delivery + if c.upgrades { + upgrades, err = c.channel.ConsumeWithContext(ctx, UpgradeQueue, "control-plane-upgrades", + false, false, false, false, nil) + if err != nil { + return err + } + } + if c.catchups { + catchups, err = c.channel.ConsumeWithContext(ctx, CatchUpQueue, "control-plane-catchup", + false, false, false, false, nil) + if err != nil { + return err + } + } + + closed := c.conn.NotifyClose(make(chan *amqp.Error, 1)) + + again := c.again + if again == 0 { + again = TryAgainAfter + } + ticker := time.NewTicker(again) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if ctx.Err() != nil { + return nil + } + c.retryHeld(ctx, act) + case delivery, ok := <-catchups: + if !ok { + if catchups != nil { + return errors.New("the bus stopped delivering catch-up requests") + } + continue + } + act(ctx, c.wrap(KindCatchUp, delivery)) + case delivery, ok := <-upgrades: + // A nil channel blocks for ever, so this case simply never fires when nothing is + // listening for upgrades. Closed is different, and means the bus stopped. + if !ok { + if upgrades != nil { + return errors.New("the bus stopped delivering upgrades") + } + continue + } + act(ctx, c.wrap(KindModuleMoved, delivery)) + case reason := <-closed: + // Said rather than returned quietly. A controller whose bus connection dropped is a + // mesh where nothing can be told anything, and the reason is the first thing anybody + // will want. + return fmt.Errorf("the bus connection closed: %v", reason) + case delivery, ok := <-deliveries: + if !ok { + return errors.New("the bus stopped delivering") + } + kind, known := kindOfKey[delivery.RoutingKey] + if !known { + // Rejected without requeue: a message nothing understands will not be understood + // on the next attempt either, and requeuing it would spin. + _ = delivery.Reject(false) + continue + } + act(ctx, c.wrap(kind, delivery)) + } + } +} + +// kindOfKey is how this transport's addressing becomes what the mesh calls a message. +var kindOfKey = map[string]string{ + KeyEnrol: KindEnrolment, + KeyReport: KindReport, + KeyAlive: KindHeartbeat, + KeyBuilt: KindBuilt, + KeyModuleUpgraded: KindModuleMoved, + KeyCatchingUp: KindCatchUp, +} + +func (c *currentInbound) wrap(kind string, delivery amqp.Delivery) *currentControl { + return ¤tControl{kind: kind, delivery: delivery, on: c} +} + +// retryHeld hands every message whose delay has passed back to the loop. Each handler holds it +// again, settles it, or lets it go past the bound. +func (c *currentInbound) retryHeld(ctx context.Context, act func(context.Context, Control)) { + now := time.Now() + due := make([]*currentControl, 0, len(c.held)) + for _, h := range c.held { + if !h.due.After(now) { + due = append(due, h.message) + } + } + for _, m := range due { + if ctx.Err() != nil { + return + } + act(ctx, m) + } +} + +// currentControl is one delivery from the bus the mesh has, as the controller reads it. +type currentControl struct { + kind string + delivery amqp.Delivery + on *currentInbound + // about is what this message is about, as the handler named it; empty until it does. + about string + // first is when this message was first held for the store; zero while it has not been. + first time.Time +} + +func (m *currentControl) Kind() string { return m.kind } +func (m *currentControl) Body() []byte { return m.delivery.Body } +func (m *currentControl) Redelivered() bool { return m.delivery.Redelivered } + +func (m *currentControl) HeldFor() time.Duration { + if m.first.IsZero() { + return 0 + } + return time.Since(m.first) +} + +// Answer publishes to the reply queue the request named. +func (m *currentControl) Answer(ctx context.Context, body []byte) error { + if m.delivery.ReplyTo == "" { + return errors.New("that request named no reply queue, so nothing can be told the answer") + } + return m.on.channel.PublishWithContext(ctx, "", m.delivery.ReplyTo, false, false, + amqp.Publishing{ + ContentType: "application/json", + CorrelationId: m.delivery.CorrelationId, + Body: body, + }) +} + +func (m *currentControl) Took() error { + m.forget() + return m.delivery.Ack(false) +} + +// Drop rejects without requeue: on this bus that is what "understood, and not worth another +// attempt" is spelled as, and it is what feeds a dead-letter queue where one is configured. +func (m *currentControl) Drop() error { + m.forget() + return m.delivery.Reject(false) +} + +// About names what this message is about, and lets go of whatever is held about the same thing: +// the held one is the past, and acting on it after this one would undo this one. Acknowledged +// rather than left to come back, because a held message nothing will act on is a place in the +// prefetch nothing gets back. +func (m *currentControl) About(what string) { + m.about = what + if what == "" { + return + } + for tag, h := range m.on.held { + if h.about != what || tag == m.delivery.DeliveryTag { + continue + } + delete(m.on.held, tag) + _ = h.message.delivery.Ack(false) + } +} + +// Hold keeps the message unacknowledged and sets it aside to be handed back after the delay. +// +// Held no further than the prefetch leaves room: past that the bus would hand the loop nothing +// new — enrolments included — until something held was let go. A message that cannot be held says +// so, and the handler settles it its own way. +func (m *currentControl) Hold(after time.Duration) error { + if m.on.held == nil { + m.on.held = map[uint64]*holding{} + } + if _, already := m.on.held[m.delivery.DeliveryTag]; !already { + if len(m.on.held) >= Prefetch-PrefetchHeadroom { + return fmt.Errorf("%d messages are already held for the store, and holding more "+ + "would stop the queue", len(m.on.held)) + } + m.first = time.Now() + } + m.on.held[m.delivery.DeliveryTag] = &holding{ + message: m, due: time.Now().Add(after), about: m.about, + } + return nil +} + +func (m *currentControl) forget() { + if m.on != nil { + delete(m.on.held, m.delivery.DeliveryTag) + } +} diff --git a/internal/link/receive_current_test.go b/internal/link/receive_current_test.go new file mode 100644 index 0000000..462e3f3 --- /dev/null +++ b/internal/link/receive_current_test.go @@ -0,0 +1,71 @@ +package link + +import ( + "context" + "encoding/json" + "io" + "log" + "testing" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +// The harness for the consume side on the bus the mesh runs on today. +// +// Messages arrive through the seam, so what these tests exercise is the controller's decision +// about a message and this transport's way of keeping one — which is what the seam separated. A +// fake acknowledger stands in for the bus, because what is asserted is how a message was settled +// and that needs no server. + +// settled is how the bus was told to settle one message. +type settled struct{ acked, nacked, requeued, rejected bool } + +func (a *settled) Ack(uint64, bool) error { a.acked = true; return nil } +func (a *settled) Nack(_ uint64, _ bool, requeue bool) error { + a.nacked, a.requeued = true, requeue + return nil +} +func (a *settled) Reject(uint64, bool) error { a.rejected = true; return nil } + +// unsettled is a message the controller has neither taken nor let go: it is held, and the bus will +// hand it to whatever consumes next if the controller stops. +func (a *settled) unsettled() bool { return !a.acked && !a.nacked && !a.rejected } + +var tag uint64 + +func quiet() *log.Logger { return log.New(io.Discard, "", 0) } + +// serving is a controller with nothing but a way of receiving, ready for a listener, a recorder, +// an upgrader or a replayer to be set on it. +func serving() (*Server, *currentInbound) { + in := ¤tInbound{held: map[uint64]*holding{}} + return &Server{inbound: in, bus: OverCurrent{}, log: quiet()}, in +} + +// sends is one message arriving over this transport, as the controller reads it. +func (c *currentInbound) sends(t *testing.T, to *settled, kind string, v any) Control { + t.Helper() + body, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + tag++ + return ¤tControl{kind: kind, on: c, delivery: amqp.Delivery{ + Acknowledger: to, Body: body, DeliveryTag: tag, + }} +} + +// dueNow brings every held message forward, so a test need not wait out the backoff a real store +// restart would be given (RedeliverAfter). +func (c *currentInbound) dueNow() { + for _, h := range c.held { + h.due = time.Now().Add(-time.Second) + } +} + +// retries hands every held message back to the controller, the way the ticker does. +func (c *currentInbound) retries(ctx context.Context, s *Server) { + c.dueNow() + c.retryHeld(ctx, s.act) +} diff --git a/internal/link/receive_nats.go b/internal/link/receive_nats.go new file mode 100644 index 0000000..24ab322 --- /dev/null +++ b/internal/link/receive_nats.go @@ -0,0 +1,290 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The consume side on the bus being built. +// +// The shape is the AMQP one's, because the seam made them comparable: one loop, one message at a +// time, and the same window deciding. What differs is where a held message lives — and that is the +// whole point of the move. On the bus the mesh has, holding one means keeping an unacknowledged +// delivery in this process, bounded by the prefetch and lost if the controller stops. Here it is a +// `nak` with a delay: the message stays the server's, the controller keeps nothing but the moment +// it first could not take it, and a controller that restarts mid-window has nothing to lose. + +// natsInbound consumes what nodes and modules say over NATS. +type natsInbound struct { + js *broker.JetStream + // follows is the kinds asked for beyond what nodes say (Also). The events those are are the + // only ones the controller subscribes, and only when something is listening. + follows map[string]bool + // since is when the controller first could not take a message, by that message's place in its + // stream. + // + // **A timestamp, not a message.** This is the whole difference the move buys: the AMQP side + // keeps the delivery, and this keeps eight bytes saying when the window opened. A controller + // that restarts loses these and starts the window again, which is correct — it is holding + // nothing, and the messages are all still on the server. + since map[uint64]time.Time +} + +// Nats is the consume side of the bus being built. +func Nats(js *broker.JetStream) Inbound { + return &natsInbound{js: js, follows: map[string]bool{}, since: map[uint64]time.Time{}} +} + +// Also records one more kind to subscribe. Nothing is subscribed here: the controller's consumer on +// the events stream carries both of these as filters, so it is created once, in Receive, with +// whatever was asked for — and not at all when nothing was. +func (n *natsInbound) Also(kind string) error { + switch kind { + case KindModuleMoved, KindCatchUp: + n.follows[kind] = true + return nil + default: + return fmt.Errorf("nothing subscribes %s separately on this bus", kind) + } +} + +func (n *natsInbound) Close() {} + +// Receive consumes until the context ends. +// +// Three subscriptions, and each is a channel the one loop selects on. **Channels rather than +// callbacks**: the library would run a handler on its own goroutine, and the window's bookkeeping — +// which message is held, and since when — is read and written without a lock because the AMQP loop +// never had two. A second goroutine would make that wrong in a way no test would catch. +func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Control)) error { + if err := broker.AssertMeshConsumers(n.js); err != nil { + return err + } + js, conn := n.js.Context(), n.js.Conn() + + // What nodes say, off the CONTROL stream. Bound to the durable the controller asserted rather + // than creating one here: the consumer is an object with a configuration — ack policy, ack + // wait, redelivery — and a client that creates its own would be a second opinion about it. + control := make(chan *nats.Msg, Prefetch) + said, err := js.ChanSubscribe("", control, nats.Bind("CONTROL", broker.ControllerName)) + if err != nil { + return fmt.Errorf("subscribing to what nodes say: %w", err) + } + defer func() { _ = said.Unsubscribe() }() + + // Heartbeats, on core NATS and off any stream (design 25 §3). Their own subscription because + // they are their own guarantee: a lost one is the next one. + beats := make(chan *nats.Msg, Prefetch) + alive, err := conn.ChanSubscribe(AliveSubjects, beats) + if err != nil { + return fmt.Errorf("subscribing to heartbeats: %w", err) + } + defer func() { _ = alive.Unsubscribe() }() + + // The events the controller follows, when something is listening for them. + var events chan *nats.Msg + if len(n.follows) > 0 { + events = make(chan *nats.Msg, Prefetch) + followed, err := js.ChanSubscribe("", events, nats.Bind("EVENTS", broker.ControllerName)) + if err != nil { + return fmt.Errorf("subscribing to what the catalogue says: %w", err) + } + defer func() { _ = followed.Unsubscribe() }() + } + + // A connection that dropped is said, not discovered. A controller whose bus connection is gone + // is a mesh where nothing can be told anything. + gone := make(chan error, 1) + conn.SetDisconnectErrHandler(func(_ *nats.Conn, err error) { + select { + case gone <- err: + default: + } + }) + + for { + select { + case <-ctx.Done(): + return nil + case err := <-gone: + return fmt.Errorf("the bus connection dropped: %w", err) + case msg := <-beats: + n.deliver(ctx, act, msg, false) + case msg := <-events: + n.deliver(ctx, act, msg, true) + case msg, ok := <-control: + if !ok { + return errors.New("the bus stopped delivering") + } + n.deliver(ctx, act, msg, true) + } + } +} + +// deliver names one message and hands it to the loop, or drops it where the mesh has no name for +// its subject — which cannot happen through a filter the controller wrote, and is said rather than +// ignored for exactly that reason. +func (n *natsInbound) deliver(ctx context.Context, act func(context.Context, Control), + msg *nats.Msg, streamed bool) { + + kind, known := kindOfSubject(msg.Subject) + if !known { + if streamed { + _ = msg.Term() + } + return + } + m := &natsControl{kind: kind, msg: msg, on: n} + if streamed { + // A message with no metadata is not from a stream, whatever it was delivered on, and the + // window has nothing to hold it by. Said by leaving the sequence at zero. + if meta, err := msg.Metadata(); err == nil { + m.seq = meta.Sequence.Stream + m.delivered = meta.NumDelivered + } + } + act(ctx, m) +} + +// kindOfSubject is how this transport's addressing becomes what the mesh calls a message. +// +// By subject, which is the only thing the server enforces: a body claiming to be a report does not +// make it one, and on this bus the subject an account may publish *is* its authority (design 29 +// §2). The mirror of the routing-key table on the bus the mesh has. +func kindOfSubject(subject string) (string, bool) { + switch subject { + case EnrolSubject: + return KindEnrolment, true + case BuiltSubject: + return KindBuilt, true + } + if node, rest, ok := strings.Cut(strings.TrimPrefix(subject, "mesh.control."), "."); ok && + node != "" && !strings.Contains(node, ".") { + switch rest { + case "report": + return KindReport, true + case "alive": + return KindHeartbeat, true + } + } + switch subject { + case broker.ControllerFollows[0]: + return KindModuleMoved, true + case broker.ControllerFollows[1]: + return KindCatchUp, true + case BuildOutcome(): + // A build's outcome is the role's event now, so it arrives on the events stream rather than + // the control branch — and is acted on by the same handler, because what the controller does + // with it did not change (novox/hq ADR 0121). + return KindBuilt, true + } + return "", false +} + +// natsControl is one message from the bus being built, as the controller reads it. +type natsControl struct { + kind string + msg *nats.Msg + on *natsInbound + // seq is this message's place in its stream; zero for a core message, which has none and + // cannot be held. + seq uint64 + // delivered is how many times the server has handed this message over, this time included. + delivered uint64 +} + +func (m *natsControl) Kind() string { return m.kind } +func (m *natsControl) Body() []byte { return m.msg.Data } + +// Redelivered is what the server counted, not what the controller remembers. Which is the answer to +// a question the AMQP side could only guess at across a restart: an enrolment redelivered because +// the controller stopped mid-answer reads as redelivered to the controller that comes back. +func (m *natsControl) Redelivered() bool { return m.delivered > 1 } + +func (m *natsControl) HeldFor() time.Duration { + if m.seq == 0 { + return 0 + } + first, held := m.on.since[m.seq] + if !held { + return 0 + } + return time.Since(first) +} + +// About is nothing here, and that is the point. +// +// Setting a held message aside when a newer one about the same thing arrives is what a controller +// holding deliveries in memory can do. A naked message belongs to the server and comes back +// whatever happened meanwhile, so the question "is this the past?" is answered by what the message +// says instead — the digest of the declaration a report is about (window.go, design 25 §3). +func (m *natsControl) About(string) {} + +// Answer publishes to the reply subject the request carries **in its payload**. +// +// Not `Respond`, and not the message's reply field: a message a JetStream consumer delivers has had +// that field claimed for the consumer's own ack address, so answering it would send the reply to +// `$JS.ACK.CONTROL.controller.…` and the enrolling node would wait out its timeout. Verified +// against a running server (design 25 §2), which is why it is a field of the request and this reads +// it from there. +func (m *natsControl) Answer(ctx context.Context, body []byte) error { + var addressed replyAddressed + if err := json.Unmarshal(m.msg.Data, &addressed); err != nil { + return fmt.Errorf("that request cannot be read, so its reply address cannot be: %w", err) + } + if addressed.ReplyTo == "" { + return errors.New("that request named no reply subject in its payload, so nothing can be " + + "told the answer") + } + return m.on.js.Conn().PublishMsg(&nats.Msg{Subject: addressed.ReplyTo, Data: body}) +} + +func (m *natsControl) Took() error { + m.forget() + if m.seq == 0 { + // Core NATS: nothing is keeping it, so there is nothing to settle. + return nil + } + return m.msg.Ack(nats.Context(context.Background())) +} + +// Drop terminates the delivery: understood, and the server is told not to send it again. Different +// from an ack only in the server's own accounting, which is where somebody asking "what happened to +// that message" will look. +func (m *natsControl) Drop() error { + m.forget() + if m.seq == 0 { + return nil + } + return m.msg.Term() +} + +// Hold hands the message back with a delay, and remembers when the window opened. +func (m *natsControl) Hold(after time.Duration) error { + if m.seq == 0 { + return errors.New("a message that is not in a stream cannot be held: nothing is keeping it") + } + if _, already := m.on.since[m.seq]; !already { + m.on.since[m.seq] = time.Now() + } + return m.msg.NakWithDelay(after) +} + +func (m *natsControl) forget() { + if m.on != nil && m.seq != 0 { + delete(m.on.since, m.seq) + } +} + +// replyAddressed is the one field every message that expects an answer carries. +type replyAddressed struct { + ReplyTo string `json:"reply_to,omitempty"` +} diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go new file mode 100644 index 0000000..0dc2edf --- /dev/null +++ b/internal/link/receive_nats_test.go @@ -0,0 +1,376 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" +) + +// The consume side against a real server, because what is being checked is what the server does. +// +// Reasoning cannot answer any of these: whether a nak-with-delay really comes back, whether the +// delay is honoured, whether terminating a delivery really stops it, or whether an answer published +// to an address carried in the payload reaches a caller waiting on its own inbox. Each is a claim +// about a server, so each is asked of one: +// +// docker run -d --rm --name t -p 14222:4222 nats:2.10-alpine -js +// MESH_TEST_NATS=nats://127.0.0.1:14222 go test ./internal/link/ -run TestNats + +func aBus(t *testing.T) *broker.JetStream { + t.Helper() + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + + // **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong. + // + // The streams are emptied rather than deleted and recreated, because delete-then-add is not a + // reset: the server's teardown races the creation, and a test then inherits the previous one's + // messages — which reads as a redelivery bug in the code under test. + // + // The consumers are removed, because deleting a stream used to take them with it and purging + // does not. A durable *push* consumer that survives between tests keeps pushing to a delivery + // subject the previous test's subscription has gone from: the messages count as delivered, go + // nowhere, and the next test waits out its timeout for an announcement the server believes it + // already sent. The controller recreates what it needs on start, so leaving none is correct. + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + clean := func() { + for _, c := range broker.MeshConsumers() { + _ = js.Context().DeleteConsumer(c.Stream, c.Name) + } + for _, s := range broker.MeshStreams() { + _ = js.Context().PurgeStream(s.Name) + } + } + clean() + t.Cleanup(clean) + return js +} + +// serving1 is a controller reading from a real bus, and a way to stop it. +func servingOn(t *testing.T, js *broker.JetStream, l Listener) (*Server, func()) { + t.Helper() + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + listener: l, log: quiet()} + ctx, stop := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = s.Serve(ctx) }() + return s, func() { + stop() + <-done + } +} + +// counted records reports and can be told to refuse them, from another goroutine. +type counted struct { + mu sync.Mutex + err error + heard []Report +} + +func (c *counted) Heard(_ context.Context, r Report) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.err != nil { + return c.err + } + c.heard = append(c.heard, r) + return nil +} + +func (c *counted) refusing(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.err = err +} + +func (c *counted) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.heard) +} + +func eventually(t *testing.T, what string, is func() bool) { + t.Helper() + deadline := time.Now().Add(8 * time.Second) + for time.Now().Before(deadline) { + if is() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s did not happen within the wait", what) +} + +// A report published by a node reaches the controller, is recorded, and is acknowledged — so the +// stream does not hold it. A work queue is the check: what is acknowledged leaves it. +func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "a report being recorded", func() bool { return store.count() == 1 }) + eventually(t, "the report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +// **The store window, in the server.** A report the store cannot take is naked with a delay and +// comes back; once the store is there it is recorded and leaves the stream. The controller holds +// nothing in the meantime — which is what the sequence check below is for: the message is still on +// the server while it waits. +func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up"))) + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + + // Held: the message is the server's, unacknowledged, and still in the stream. + eventually(t, "the report being redelivered at least once", func() bool { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + return err == nil && info.NumRedelivered >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 1 { + t.Fatalf("a held report did not stay on the server: %+v, %v", info, err) + } + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that was refusing it") + } + + store.refusing(nil) + eventually(t, "the report being recorded once the store was back", + func() bool { return store.count() == 1 }) + eventually(t, "the recorded report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +// A report about a declaration the mesh has moved past is settled without being acted on, and +// leaves the stream rather than coming back for ever. +func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { + js := aBus(t) + store := &sentAndHeardSafely{sent: "d2"} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the superseded report leaving the stream", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report about a superseded declaration was acted on") + } +} + +// sentAndHeardSafely is sentAndHeard, read from two goroutines. +type sentAndHeardSafely struct { + mu sync.Mutex + sent string + heard []Report +} + +func (s *sentAndHeardSafely) Heard(_ context.Context, r Report) error { + s.mu.Lock() + defer s.mu.Unlock() + s.heard = append(s.heard, r) + return nil +} + +func (s *sentAndHeardSafely) Outstanding(context.Context, string) (string, error) { + return s.sent, nil +} + +func (s *sentAndHeardSafely) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.heard) +} + +// **An enrolment answered through a reply address the stream would have eaten.** +// +// The caller waits on its own inbox and states that address in the request's payload. The check is +// that the answer arrives there — which is the whole reason the address is a field rather than the +// transport's reply, and this is the test design 25 §2 asks for so the reason cannot quietly become +// folklore. +func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { + js := aBus(t) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + inbox := nats.NewInbox() + answers, err := js.Conn().SubscribeSync(inbox) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox}) + if _, err := js.Context().Publish(EnrolSubject, body); err != nil { + t.Fatal(err) + } + + msg, err := answers.NextMsg(8 * time.Second) + if err != nil { + t.Fatalf("no answer reached the address the request named: %v", err) + } + var reply EnrolReply + if err := json.Unmarshal(msg.Data, &reply); err != nil { + t.Fatal(err) + } + if !reply.Accepted || reply.Node != "anchor" { + t.Fatalf("the answer was not the mesh's: %+v", reply) + } + // And the address really is not the one the transport carried: what the consumer saw was its + // own ack subject, which is why this had to travel in the payload. + if msg.Subject != inbox { + t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject) + } +} + +// enrolsAs answers every request the same way. +type enrolsAs struct{ reply EnrolReply } + +func (e enrolsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) { return e.reply, nil } + +// A heartbeat is core NATS: it reaches the controller and nothing is persisted, so the stream the +// reports live in stays empty. +func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + // Given time to subscribe: a core subscription that is not yet up misses what is published, + // which is the guarantee a heartbeat has and not a fault. + eventually(t, "the heartbeat subscription coming up", func() bool { + body, _ := json.Marshal(Alive{Node: "anchor"}) + _ = js.Conn().Publish(AliveSubject("anchor"), body) + _ = js.Conn().Flush() + return store.count() >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 0 { + t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+ + "for retention with its most valuable: %+v, %v", info, err) + } +} + +// The two events the controller follows arrive over one durable consumer with two filters, and it +// can acknowledge them. +// +// **Both halves are the point.** A consumer with several filter subjects is a 2.10 feature and this +// is the first thing in the mesh to use one; and a delivery from the events stream is acknowledged +// on a different ack subject from a delivery from the control stream, which the controller's own +// permission list has to cover or every announcement is redelivered for ever. +func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) { + js := aBus(t) + told := &toldAbout{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(told); err != nil { + t.Fatal(err) + } + if err := s.Answers(replaysWith{}); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil { + t.Fatal(err) + } + + eventually(t, "the catalogue's upgrade reaching the controller", + func() bool { return told.count() == 1 }) + eventually(t, "both announcements being acknowledged", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 + }) +} + +type toldAbout struct { + mu sync.Mutex + saw []Upgraded + fail error +} + +func (u *toldAbout) Upgraded(_ context.Context, m Upgraded) error { + u.mu.Lock() + defer u.mu.Unlock() + if u.fail != nil { + return u.fail + } + u.saw = append(u.saw, m) + return nil +} + +func (u *toldAbout) count() int { + u.mu.Lock() + defer u.mu.Unlock() + return len(u.saw) +} + +// A store that never comes back: the report is let go once the bound passes, and it leaves the +// stream rather than being held for ever. The bound is the controller's, not the server's — nothing +// here sets max-deliver, and that is deliberate (streams.go). +func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused"))) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + listener: store, log: quiet(), giveUp: 1500 * time.Millisecond} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the report being let go once the bound passed", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that never came back") + } +} diff --git a/internal/link/report_retry_test.go b/internal/link/report_retry_test.go index 8a175d7..7549437 100644 --- a/internal/link/report_retry_test.go +++ b/internal/link/report_retry_test.go @@ -2,26 +2,11 @@ package link import ( "context" - "encoding/json" "errors" - "io" - "log" "testing" "time" - - amqp "github.com/rabbitmq/amqp091-go" ) -type saidTo struct{ acked, nacked, requeued bool } - -func (a *saidTo) Ack(uint64, bool) error { a.acked = true; return nil } -func (a *saidTo) Nack(_ uint64, _ bool, requeue bool) error { - a.nacked, a.requeued = true, requeue - return nil -} -func (a *saidTo) Reject(uint64, bool) error { return nil } -func (a *saidTo) unsettled() bool { return !a.acked && !a.nacked } - type heardWith struct{ err error } func (h heardWith) Heard(context.Context, Report) error { return h.err } @@ -31,39 +16,31 @@ type switchable struct{ err error } func (h *switchable) Heard(context.Context, Report) error { return h.err } -var tag uint64 - -func aReport(t *testing.T, to *saidTo, node, declared string) amqp.Delivery { - t.Helper() - body, err := json.Marshal(Report{Node: node, Declared: declared, Applied: []string{"store"}}) - if err != nil { - t.Fatal(err) - } - tag++ - return amqp.Delivery{Acknowledger: to, RoutingKey: KeyReport, Body: body, DeliveryTag: tag} +func aReport(node, declared string) Report { + return Report{Node: node, Declared: declared, Applied: []string{"store"}} } -func quiet() *log.Logger { return log.New(io.Discard, "", 0) } - // A report the store could not take right now is held, unsettled, and recorded when the store is // back; one the store answered no to is acknowledged; one recorded is acknowledged (issue 082, 083). func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) { store := &switchable{err: errors.Join(ErrTryAgain, errors.New("starting up"))} - s := &Server{listener: store, log: quiet()} - held := &saidTo{} - s.handleReport(context.Background(), aReport(t, held, "anchor", "d1")) - if !held.unsettled() || len(s.parked) != 1 { - t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(s.parked)) + s, in := serving() + s.listener = store + held := &settled{} + s.act(context.Background(), in.sends(t, held, KindReport, aReport("anchor", "d1"))) + if !held.unsettled() || len(in.held) != 1 { + t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(in.held)) } store.err = nil - s.retryHeld(context.Background()) - if !held.acked || len(s.parked) != 0 { - t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(s.parked)) + in.retries(context.Background(), s) + if !held.acked || len(in.held) != 0 { + t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(in.held)) } - refused := &saidTo{} - s = &Server{listener: heardWith{err: errors.New("a report named no node")}, log: quiet()} - s.handleReport(context.Background(), aReport(t, refused, "anchor", "d1")) + refused := &settled{} + s, in = serving() + s.listener = heardWith{err: errors.New("a report named no node")} + s.act(context.Background(), in.sends(t, refused, KindReport, aReport("anchor", "d1"))) if !refused.acked || refused.nacked { t.Fatalf("a report the store answered no to was not acknowledged: %+v", refused) } @@ -72,33 +49,35 @@ func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) { // A newer report from the same node supersedes one of its reports still held: recorded after the // newer, the older would overwrite what the node is doing now. func TestANewerReportSupersedesAHeldOneFromTheSameNode(t *testing.T) { - s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))}, log: quiet()} - older, newer, other := &saidTo{}, &saidTo{}, &saidTo{} - s.handleReport(context.Background(), aReport(t, older, "anchor", "d1")) - s.handleReport(context.Background(), aReport(t, other, "laptop", "d7")) - s.handleReport(context.Background(), aReport(t, newer, "anchor", "d2")) + s, in := serving() + s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))} + older, newer, other := &settled{}, &settled{}, &settled{} + s.act(context.Background(), in.sends(t, older, KindReport, aReport("anchor", "d1"))) + s.act(context.Background(), in.sends(t, other, KindReport, aReport("laptop", "d7"))) + s.act(context.Background(), in.sends(t, newer, KindReport, aReport("anchor", "d2"))) if !older.acked { t.Fatalf("the older report was not set aside by the newer: %+v", older) } - if !newer.unsettled() || !other.unsettled() || len(s.parked) != 2 { + if !newer.unsettled() || !other.unsettled() || len(in.held) != 2 { t.Fatalf("the newer report and another node's were not both held: newer %+v other %+v, %d held", - newer, other, len(s.parked)) + newer, other, len(in.held)) } } // A store that has not come back within the bound is not restarting: the report is let go, loudly, // rather than held for ever. func TestAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { - s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))}, - log: quiet(), giveUp: time.Millisecond} - held := &saidTo{} - s.handleReport(context.Background(), aReport(t, held, "anchor", "d1")) + s, in := serving() + s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))} + s.giveUp = time.Millisecond + held := &settled{} + s.act(context.Background(), in.sends(t, held, KindReport, aReport("anchor", "d1"))) if !held.unsettled() { t.Fatalf("the first failure was not held: %+v", held) } time.Sleep(5 * time.Millisecond) - s.retryHeld(context.Background()) - if !held.acked || len(s.parked) != 0 { - t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(s.parked)) + in.retries(context.Background(), s) + if !held.acked || len(in.held) != 0 { + t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(in.held)) } } diff --git a/internal/link/serve.go b/internal/link/serve.go index 925c237..f1c5750 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -7,31 +7,30 @@ import ( "encoding/json" "errors" "fmt" - "github.com/novox/mesh-controller/internal/envfile" - "github.com/novox/mesh-controller/internal/inventory" "log" "os" "time" amqp "github.com/rabbitmq/amqp091-go" + + "github.com/novox/mesh-controller/internal/envfile" ) -// AMQPVar is the control plane's own connection to the broker. +// AMQPVar is the controller's own connection to the bus the mesh runs on today. const AMQPVar = "MESH_BROKER_AMQP" -// Enroller is what the control plane does with an enrolment request. +// Enroller is what the controller does with an enrolment request. // -// An interface so the serving loop can be tested against a real broker without a database, and -// so the two concerns — moving messages, and deciding — stay apart. +// An interface so the serving loop can be tested against a real bus without a database, and so the +// two concerns — moving messages, and deciding — stay apart. type Enroller interface { // Enrol spends the token, records the key, and reports the node's name. The error is // returned to the node as a refusal; it must be the same for every reason a token can fail. Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, error) } -// Server consumes what nodes say. -// Listener is what the control plane does with a report. Separate from Enroller so the two can -// be given independently, and so a server that only sends declarations needs neither. +// Listener is what the controller does with a report. Separate from Enroller so the two can be +// given independently, and so a server that only sends declarations needs neither. type Listener interface { Heard(ctx context.Context, report Report) error } @@ -46,106 +45,77 @@ type Recorder interface { Built(ctx context.Context, result BuildResult) error } -type Server struct { - conn *amqp.Connection - channel *amqp.Channel - enroller Enroller - listener Listener - recorder Recorder - log *log.Logger - upgrader Upgrader - replayer Replayer - // Messages the store could not take right now, held unacknowledged and tried again on a - // ticker, by subject (novox/hq issues 082, 083). again is the ticker's interval, zero meaning - // TryAgainAfter; giveUp is how long one is kept, zero meaning GiveUpAfter. - again time.Duration - giveUp time.Duration - parked map[string]*held -} - -// held is one message the store could not take, kept to be tried again. -type held struct { - delivery amqp.Delivery - retry func(context.Context, amqp.Delivery) - what string - first time.Time -} - -// ErrTryAgain marks a listener's failure as "not now": what it was given is worth keeping and -// asking again, as when the store is restarting (novox/hq issue 082). -var ErrTryAgain = errors.New("not now, try again") - -// TryAgainAfter is how often messages the store could not take are tried again. A store comes -// back in seconds, and a report a few seconds late is still current. -const TryAgainAfter = 2 * time.Second - -// GiveUpAfter bounds how long one message is kept trying. A store that has not come back in this -// long is not restarting, and the message is let go with a line saying it was lost. -const GiveUpAfter = 2 * time.Minute - -// Prefetch is how many messages the broker hands the control plane before it has settled them. -// More than one because a message the store could not take is held, unsettled, while the loop goes -// on answering others — an enrolment above all, which a host is waiting on (novox/hq issue 083). -// Bounded, because what is held is also what the broker has not kept on its own disk as pending. -const Prefetch = 64 - -// PrefetchHeadroom is how much of the prefetch is never held, so the loop always has messages to -// answer — an enrolment above all — while others wait for the store. -const PrefetchHeadroom = 8 - -// Records tells the server where to keep build results. -// -// Set after Connect rather than passed to it, because a control plane that only publishes — the -// `build` command, which waits for its own answer — needs a connection and no recorder, and -// making it supply one would have it construct something it never uses. -func (s *Server) Records(r Recorder) { s.recorder = r } - -// Upgrader is what the control plane does when the catalogue says a module moved. +// Upgrader is what the controller does when the catalogue says a module moved. // // An interface for the same reason Enroller is one: deciding what an upgrade means for the // machines running it is a different concern from noticing that one was announced, and only the // first needs a database. type Upgrader interface { // Upgraded is told which module moved and between which commits. An error is logged and the - // message is not requeued: an upgrade the control plane could not act on is not one it will - // act on by being handed the same message again, and a poison message on a durable queue - // would stop every upgrade behind it — except the store unreachable for the moment, which is - // asked again for a bounded time (novox/hq issue 083). + // message is not handed back: an upgrade the controller could not act on is not one it will + // act on by being given the same message again, and a poison message on a durable queue would + // stop every upgrade behind it — except the store unreachable for the moment, which is asked + // again for a bounded time (novox/hq issue 083). Upgraded(ctx context.Context, u Upgraded) error } -// Follows says what to do about upgrades, and binds the queue they arrive on. +// Server acts on what nodes and modules say. // -// **Not bound unless something is listening.** A durable queue bound to every upgrade with no -// consumer fills up quietly, and the first symptom is a broker out of disk rather than anything -// about modules. +// **It holds no transport.** What arrives comes through Inbound and what it publishes goes through +// Bus, so this file is the controller's *decisions* about messages and nothing about wires. The +// connection below is the bus the mesh runs on today, kept because the command line publishes over +// the same one until the rollout (ADR 0116). +type Server struct { + inbound Inbound + bus Bus + conn *amqp.Connection + channel *amqp.Channel + + enroller Enroller + listener Listener + recorder Recorder + upgrader Upgrader + replayer Replayer + + log *log.Logger + // giveUp is how long one message is held for the store; zero means GiveUpAfter. + giveUp time.Duration +} + +// ErrTryAgain marks a listener's failure as "not now": what it was given is worth keeping and +// asking again, as when the store is restarting (novox/hq issue 082). +var ErrTryAgain = errors.New("not now, try again") + +// GiveUpAfter bounds how long one message is kept trying. A store that has not come back in this +// long is not restarting, and the message is let go with a line saying it was lost. +const GiveUpAfter = 2 * time.Minute + +// Records tells the server where to keep build results. +// +// Set after Connect rather than passed to it, because a controller that only publishes — the +// `build` command, which waits for its own answer — needs a connection and no recorder, and making +// it supply one would have it construct something it never uses. +func (s *Server) Records(r Recorder) { s.recorder = r } + +// Follows says what to do about upgrades, and asks for them to be delivered. func (s *Server) Follows(u Upgrader) error { - if _, err := s.channel.QueueDeclare(UpgradeQueue, true, false, false, false, nil); err != nil { - return fmt.Errorf("cannot declare the %s queue: %w", UpgradeQueue, err) - } - if err := s.channel.QueueBind(UpgradeQueue, KeyModuleUpgraded, EventsExchange, false, nil); err != nil { - return fmt.Errorf("cannot bind %s to %s/%s: %w", UpgradeQueue, EventsExchange, KeyModuleUpgraded, err) + if err := s.inbound.Also(KindModuleMoved); err != nil { + return err } s.upgrader = u return nil } -// Answers binds the queue a catalogue's catch-up request arrives on. -// -// **Not bound unless something is listening**, for the same reason upgrades are not: a durable -// queue with no consumer fills quietly and the first symptom is a broker out of disk. +// Answers says what to do about a catalogue's catch-up request, and asks for them to be delivered. func (s *Server) Answers(r Replayer) error { - if _, err := s.channel.QueueDeclare(CatchUpQueue, true, false, false, false, nil); err != nil { - return fmt.Errorf("cannot declare the %s queue: %w", CatchUpQueue, err) - } - if err := s.channel.QueueBind(CatchUpQueue, KeyCatchingUp, EventsExchange, false, nil); err != nil { - return fmt.Errorf("cannot bind %s to %s/%s: %w", CatchUpQueue, EventsExchange, KeyCatchingUp, err) + if err := s.inbound.Also(KindCatchUp); err != nil { + return err } s.replayer = r return nil } -// Connect opens the control plane's own connection to the broker. +// Connect opens the controller's own connection to the bus. // // On the port MESH_BROKER_AMQP_PORT names when the node's settings moved the broker (novox/hq // 04-ISSUES/102) — the URL is genesis's, sealed, and its port is the one thing in it the node may @@ -164,7 +134,7 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { conn, err := amqp.Dial(url) if err != nil { - // Not quoted back: the URL carries the control plane's own broker password. + // Not quoted back: the URL carries the controller's own bus password. return nil, fmt.Errorf("cannot reach the broker named in %s: %w", AMQPVar, err) } channel, err := conn.Channel() @@ -173,17 +143,17 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { return nil, err } - // Declared here rather than assumed. The control plane is the only thing that may create - // them — a node's account can write to this exchange and read its own queue, and configure - // nothing else, so a node arriving before the control plane has ever run finds nothing and - // says so, rather than quietly creating a topology nobody designed. + // Declared here rather than assumed. The controller is the only thing that may create them — a + // node's account can write to this exchange and read its own queue, and configure nothing + // else, so a node arriving before the controller has ever run finds nothing and says so, + // rather than quietly creating a topology nobody designed. if err := channel.ExchangeDeclare(Exchange, "direct", true, false, false, false, nil); err != nil { conn.Close() return nil, fmt.Errorf("cannot declare the %s exchange: %w", Exchange, err) } - // The events exchange too. The control plane is not the only publisher on it — modules - // announce onto it with their own accounts — but it is the only thing permitted to create it, - // for the same reason it is the only thing permitted to create the direct one. + // The events exchange too. The controller is not the only publisher on it — modules announce + // onto it with their own accounts — but it is the only thing permitted to create it, for the + // same reason it is the only thing permitted to create the direct one. if err := channel.ExchangeDeclare(EventsExchange, "topic", true, false, false, false, nil); err != nil { conn.Close() return nil, fmt.Errorf("cannot declare the %s exchange: %w", EventsExchange, err) @@ -192,7 +162,7 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { conn.Close() return nil, fmt.Errorf("cannot declare the %s queue: %w", ControlQueue, err) } - // Every key a node may publish. Binding one and forgetting another is a message the broker + // Every key a node may publish. Binding one and forgetting another is a message the bus // accepts, finds no queue for, and drops — the publisher sees success and the consumer sees // nothing. That is exactly what happened to reports: `report` was left unbound while `enrol` // worked, so nodes announced what they had applied into a void for an afternoon. @@ -203,14 +173,24 @@ func Connect(enroller Enroller, listener Listener) (*Server, error) { } } - return &Server{conn: conn, channel: channel, enroller: enroller, listener: listener, - log: log.New(os.Stdout, "", log.LstdFlags)}, nil + return &Server{ + inbound: Current(conn, channel), + bus: OverCurrent{Channel: channel}, + conn: conn, + channel: channel, + enroller: enroller, + listener: listener, + log: log.New(os.Stdout, "", log.LstdFlags), + }, nil } -// Channel is the control plane's channel, for sending declarations. +// Channel is the controller's channel, for the command line's own publishing. func (s *Server) Channel() *amqp.Channel { return s.channel } func (s *Server) Close() { + if s.inbound != nil { + s.inbound.Close() + } if s.channel != nil { _ = s.channel.Close() } @@ -219,131 +199,120 @@ func (s *Server) Close() { } } -// Serve consumes until the context ends. -// -// One consumer, deliberately: with two, the broker would round-robin between them and each would -// receive half of what it expects — a fault this project has already had, between a module's -// daemon and its capability server. +// Serve acts on what arrives until the context ends. func (s *Server) Serve(ctx context.Context) error { - // A bounded prefetch rather than one. The loop still takes messages one at a time; what the - // prefetch buys is that a message the store could not take can be held while the loop goes on - // to the next, instead of every enrolment waiting behind it (novox/hq issue 083). Anything held - // goes back to the broker if the control plane stops, because nothing held is acknowledged. - if err := s.channel.Qos(Prefetch, 0, false); err != nil { - return err - } - - deliveries, err := s.channel.ConsumeWithContext(ctx, ControlQueue, "control-plane", - false, false, false, false, nil) - if err != nil { - return err - } - - // The upgrade queue, when something is listening for them. A second queue rather than a - // second consumer on the first: two consumers on one queue split its messages between them, - // which is the fault the comment above exists about. Two queues share nothing. - var upgrades <-chan amqp.Delivery + s.log.Printf("consuming what nodes say: %s, %s, %s, %s", + KindEnrolment, KindReport, KindHeartbeat, KindBuilt) if s.upgrader != nil { - upgrades, err = s.channel.ConsumeWithContext(ctx, UpgradeQueue, "control-plane-upgrades", - false, false, false, false, nil) - if err != nil { - return err - } + s.log.Printf("following %s", KindModuleMoved) } - - // Its own queue and its own consumer, for the reason above: two consumers on one queue split - // its messages, and a catch-up request going to whichever half was not listening is a gap that - // looks like a working mesh. - var catchups <-chan amqp.Delivery if s.replayer != nil { - catchups, err = s.channel.ConsumeWithContext(ctx, CatchUpQueue, "control-plane-catchup", - false, false, false, false, nil) - if err != nil { - return err - } - } - - closed := s.conn.NotifyClose(make(chan *amqp.Error, 1)) - s.log.Printf("consuming %s, bound to %s/{%s,%s,%s,%s}", - ControlQueue, Exchange, KeyEnrol, KeyReport, KeyAlive, KeyBuilt) - if s.upgrader != nil { - s.log.Printf("consuming %s, bound to %s/%s", UpgradeQueue, EventsExchange, KeyModuleUpgraded) - } - - again := s.again - if again == 0 { - again = TryAgainAfter - } - ticker := time.NewTicker(again) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - if ctx.Err() != nil { - return nil - } - s.retryHeld(ctx) - case delivery, ok := <-catchups: - if !ok { - if catchups != nil { - return errors.New("the broker stopped delivering catch-up requests") - } - continue - } - s.catchingUp(ctx, delivery) - case delivery, ok := <-upgrades: - // A nil channel blocks for ever, so this case simply never fires when nothing is - // listening for upgrades. Closed is different, and means the broker stopped. - if !ok { - if upgrades != nil { - return errors.New("the broker stopped delivering upgrades") - } - continue - } - s.upgraded(ctx, delivery) - case reason := <-closed: - // Said rather than returned quietly. A control plane whose broker connection dropped - // is a mesh where nothing can be told anything, and the reason is the first thing - // anybody will want. - return fmt.Errorf("the broker connection closed: %v", reason) - case delivery, ok := <-deliveries: - if !ok { - return errors.New("the broker stopped delivering") - } - s.handle(ctx, delivery) - } + s.log.Printf("answering %s", KindCatchUp) } + return s.inbound.Receive(ctx, s.act) } -func (s *Server) handle(ctx context.Context, delivery amqp.Delivery) { - switch delivery.RoutingKey { - case KeyEnrol: - s.handleEnrol(ctx, delivery) - case KeyReport: - s.handleReport(ctx, delivery) - case KeyAlive: - s.handleAlive(delivery) - case KeyBuilt: - s.handleBuilt(ctx, delivery) +// act is one message, whichever bus it came over. +func (s *Server) act(ctx context.Context, m Control) { + switch m.Kind() { + case KindEnrolment: + s.enrolling(ctx, m) + case KindReport: + s.reported(ctx, m) + case KindHeartbeat: + s.heartbeat(m) + case KindBuilt: + s.wasBuilt(ctx, m) + case KindModuleMoved: + s.moved(ctx, m) + case KindCatchUp: + s.catchingUp(ctx, m) default: - // Rejected without requeue: a message nothing understands will not be understood on the - // next attempt either, and requeuing it would spin. - s.log.Printf("refusing a message with routing key %q", delivery.RoutingKey) - _ = delivery.Reject(false) + // Dropped: a message nothing understands will not be understood on the next attempt + // either, and asking for it again would spin. + s.log.Printf("refusing a message the mesh has no name for: %q", m.Kind()) + _ = m.Drop() } } -// handleAlive records that a node was heard from, and nothing else. +// decide asks the window what to do about one message, and holds it when that is the answer — +// because holding is the one verdict that means the same thing everywhere. // -// Deliberately silent: a node saying it is there every minute would fill the log with the -// ordinary case, and a log where the ordinary case is loud is a log nobody reads. -func (s *Server) handleAlive(delivery amqp.Delivery) { +// The other three come back, because "settled without acting" is a build result dropped and an +// upgrade taken, and only the handler knows which its message is. Hold means the message is the +// bus's problem now and **the caller must not settle it**; that is also what a cancelled context +// gets, because shutting down is not an answer about a message (novox/hq issue 083, on review). +// +// declaredIn is the declaration this message is about, empty when it is about none; outstanding is +// what the mesh last sent that node, empty when it has sent none or could not be asked. +func (s *Server) decide(ctx context.Context, m Control, what, declaredIn, outstanding string, + err error) Verdict { + + if ctx.Err() != nil { + return Hold + } + window := StoreWindow{GiveUpAfter: s.giveUpAfter()} + switch v := window.Decide(err, declaredIn, outstanding, m.HeldFor()); v { + case Hold: + // Said once, on the first hold. Every redelivery saying it again would fill the log with + // one store restart. + first := m.HeldFor() == 0 + if held := m.Hold(RedeliverAfter(m.HeldFor())); held != nil { + s.log.Printf("LOST %s: it could not be held for the store (%v): %v", what, held, err) + return GiveUp + } + if first { + s.log.Printf("could not keep %s yet; holding it to try again: %v", what, err) + } + return Hold + case Stale: + s.log.Printf("set aside %s: the mesh has moved past that declaration", what) + return Stale + case GiveUp: + s.log.Printf("LOST %s: the store has not come back in %s: %v", what, s.giveUpAfter(), err) + return GiveUp + default: + return v + } +} + +func (s *Server) giveUpAfter() time.Duration { + if s.giveUp == 0 { + return GiveUpAfter + } + return s.giveUp +} + +// outstanding is the digest of the declaration the mesh last sent a node. +// +// Nothing is guessed when it cannot be answered: a listener that keeps no record of what was sent +// gives a report nothing to be stale against, and a store that cannot be read will hold the report +// anyway, so there is nothing for the check to decide. +func (s *Server) outstanding(ctx context.Context, node string) string { + if node == "" { + return "" + } + asks, ok := s.listener.(Outstanding) + if !ok { + return "" + } + digest, err := asks.Outstanding(ctx, node) + if err != nil { + return "" + } + return digest +} + +// heartbeat records that a node was heard from, and nothing else. +// +// Deliberately silent: a node saying it is there every minute would fill the log with the ordinary +// case, and a log where the ordinary case is loud is a log nobody reads. Not held for the store +// either — the next heartbeat is a minute away, and a heartbeat kept for two minutes to be written +// late says nothing the one after it will not say better. +func (s *Server) heartbeat(m Control) { var alive Alive - if err := json.Unmarshal(delivery.Body, &alive); err != nil || alive.Node == "" { - _ = delivery.Reject(false) + if err := json.Unmarshal(m.Body(), &alive); err != nil || alive.Node == "" { + _ = m.Drop() return } if s.listener != nil { @@ -351,34 +320,49 @@ func (s *Server) handleAlive(delivery amqp.Delivery) { s.log.Printf("could not record that %s is here: %v", alive.Node, err) } } - _ = delivery.Ack(false) + _ = m.Took() } -// handleReport records what a node says it did. +// reported records what a node says it did. // // A node states; nothing here writes anything the node claimed about itself beyond that it was // heard from. What it applied is its own account of its own machine, and the mesh keeps the last // one as a copy for recovery rather than as a source (novox/hq 09-the-node-lifecycle). -func (s *Server) handleReport(ctx context.Context, delivery amqp.Delivery) { +func (s *Server) reported(ctx context.Context, m Control) { var report Report - if err := json.Unmarshal(delivery.Body, &report); err != nil { + if err := json.Unmarshal(m.Body(), &report); err != nil { s.log.Printf("a report could not be read: %v", err) - _ = delivery.Reject(false) + _ = m.Drop() return } - // A node's newer report supersedes one of its older reports still held: the older is its - // past, and recorded after the newer it would overwrite what the node is doing now. - subject := "report " + report.Node - s.supersede(subject, delivery) + m.About("report " + report.Node) + what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared) + if s.listener != nil { - if err := s.listener.Heard(context.Background(), report); err != nil { - // Held, not acknowledged, while the store cannot take it: the node reports an apply - // once, and a report lost here is a node the mesh never hears from again — the store + // **Asked before the store, not after** (design 25 §3). A report about a declaration the + // mesh has moved past would otherwise wait out a restarting store to be written and then + // overwrite what the node is doing now — and on the bus being built, where the holding is + // the server's, it comes back after the newer was applied whatever the controller does. + outstanding := s.outstanding(ctx, report.Node) + declaredIn := staleAgainst(report) + if Superseded(declaredIn, outstanding) { + s.log.Printf("set aside %s: the mesh has moved past that declaration", what) + _ = m.Took() + return + } + + err := s.listener.Heard(context.Background(), report) + switch s.decide(ctx, m, what, declaredIn, outstanding, err) { + case Hold: + // Held, not settled, while the store cannot take it: the node reports an apply once, + // and a report lost here is a node the mesh never hears from again — the store // restarting under the adoption that node just applied lost exactly that (issue 082). - what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared) - if s.tryLater(ctx, delivery, subject, what, err, s.handle) { - return - } + return + case Stale, GiveUp: + _ = m.Took() + return + } + if err != nil { // Said rather than swallowed. A report the mesh heard and failed to write down is a // node whose recovery copy is silently older than it looks. s.log.Printf("could not record %s's report: %v", report.Node, err) @@ -393,124 +377,49 @@ func (s *Server) handleReport(ctx context.Context, delivery amqp.Delivery) { default: s.log.Printf("%s applied %d resource(s)", report.Node, len(report.Applied)) } - s.settled(subject, delivery) - _ = delivery.Ack(false) + _ = m.Took() } -// tryLater holds a message the store could not take right now, to be tried again on the ticker, -// and says whether it did (novox/hq issues 082, 083). +// staleAgainst is the declaration a report may be judged stale against, and it is empty for a +// report that is not only an account of an apply. // -// "Right now" is the store unreachable or restarting — ErrTryAgain from a listener, or an error the -// inventory reads as an outage. Anything else is an answer, and is left to the caller to settle. -// Held means unacknowledged and set aside: the loop goes on to the next message, so an enrolment a -// host is waiting on is answered while a report waits for the store. One message is held at most -// giveUpAfter; past it, it is let go with a line saying it was lost, and the caller settles it. -func (s *Server) tryLater(ctx context.Context, delivery amqp.Delivery, subject, what string, err error, - retry func(context.Context, amqp.Delivery)) bool { - // Shutting down: nothing is settled. Unsettled, the broker hands the message to whatever - // consumes next — a cancelled context is not an answer about the message (issue 083, review). - if ctx.Err() != nil { - return true +// **A report carries two different things, and only one of them is about a declaration.** What the +// node applied is; what the machine *is* — the tunnel it took over, the ports its own bundle holds, +// what an adopted node found and is keeping, a node moving its overlay key — is not. Those reach +// the mesh on a report because a report is the message a node sends, and nowhere else: a rekey +// dropped as stale is a node whose overlay key never moves, and no retry is coming, because the +// node said it once. +// +// So staleness is asked only of a report that is purely an apply's account. The rest is acted on +// whenever it arrives, which is the behaviour the mesh has had all along. +func staleAgainst(report Report) string { + if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 || + report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 { + return "" } - if !errors.Is(err, ErrTryAgain) && !inventory.Unreachable(err) { - return false - } - if s.parked == nil { - s.parked = map[string]*held{} - } - h, ok := s.parked[subject] - if !ok || h.delivery.DeliveryTag != delivery.DeliveryTag { - // Held no further than the prefetch leaves room: past it, the broker would hand the loop - // nothing new — enrolments included — until something held was let go. - if !ok && len(s.parked) >= Prefetch-PrefetchHeadroom { - s.log.Printf("LOST %s: %d messages are already held for the store, and holding more "+ - "would stop the queue: %v", what, len(s.parked), err) - return false - } - h = &held{delivery: delivery, retry: retry, what: what, first: time.Now()} - s.parked[subject] = h - s.log.Printf("could not keep %s yet; holding it to try again: %v", what, err) - return true - } - if time.Since(h.first) >= s.giveUpAfter() { - delete(s.parked, subject) - s.log.Printf("LOST %s: the store has not come back in %s: %v", what, s.giveUpAfter(), err) - return false - } - return true + return report.Declared } -// supersede drops a message held for a subject when a newer one for it arrives: the older is -// acknowledged, because acting on it after the newer would undo the newer. -func (s *Server) supersede(subject string, newer amqp.Delivery) { - h, ok := s.parked[subject] - if !ok || h.delivery.DeliveryTag == newer.DeliveryTag { - return - } - delete(s.parked, subject) - s.log.Printf("set aside %s: a newer one arrived", h.what) - _ = h.delivery.Ack(false) -} - -// settled forgets a message once it has been handled either way. -func (s *Server) settled(subject string, delivery amqp.Delivery) { - if h, ok := s.parked[subject]; ok && h.delivery.DeliveryTag == delivery.DeliveryTag { - delete(s.parked, subject) - } -} - -// digest names a message by its content. -func digest(body []byte) string { - sum := sha256.Sum256(body) - return hex.EncodeToString(sum[:]) -} - -// retryHeld tries every held message again. Each handler holds it again, settles it, or lets it -// go past the bound. -func (s *Server) retryHeld(ctx context.Context) { - for _, h := range s.snapshot() { - if ctx.Err() != nil { - return - } - h.retry(ctx, h.delivery) - } -} - -func (s *Server) snapshot() []*held { - out := make([]*held, 0, len(s.parked)) - for _, h := range s.parked { - out = append(out, h) - } - return out -} - -func (s *Server) giveUpAfter() time.Duration { - if s.giveUp == 0 { - return GiveUpAfter - } - return s.giveUp -} - -func (s *Server) handleEnrol(ctx context.Context, delivery amqp.Delivery) { +func (s *Server) enrolling(ctx context.Context, m Control) { reply := EnrolReply{Refusal: "that token cannot be used"} var request EnrolRequest - if err := json.Unmarshal(delivery.Body, &request); err != nil { + if err := json.Unmarshal(m.Body(), &request); err != nil { s.log.Printf("an enrolment request could not be read: %v", err) } else { - request.Redelivered = delivery.Redelivered + request.Redelivered = m.Redelivered() accepted, err := s.enroller.Enrol(ctx, request) switch { case errors.Is(err, ErrTryAgain): // Not a refusal: nothing was spent, and the same request asked again will be - // answered. Replied at once rather than held, so the node — which is waiting on - // this answer — decides when to ask, and the queue behind it moves (issue 083). + // answered. Replied at once rather than held, so the node — which is waiting on this + // answer — decides when to ask, and the queue behind it moves (issue 083). reply = EnrolReply{TryAgain: true, Refusal: "the mesh cannot answer right now; ask again"} s.log.Printf("asked %q to enrol again shortly: %v", request.Node, err) case err != nil && request.Redelivered: - // Said as what it most likely is: the broker handed this request over again after - // the control plane stopped mid-answer, and an enrolment already spent is not - // finished a second time. The node may need a new token. + // Said as what it most likely is: the bus handed this request over again after the + // controller stopped mid-answer, and an enrolment already spent is not finished a + // second time. The node may need a new token. s.log.Printf("refusing a redelivered enrolment for %q — it may have finished before "+ "the control plane stopped, and if the node did not get its answer it needs a new "+ "token: %v", request.Node, err) @@ -524,169 +433,160 @@ func (s *Server) handleEnrol(ctx context.Context, delivery amqp.Delivery) { } } - s.reply(ctx, delivery, reply) - - // Acknowledged after the reply is sent, so a control plane that dies mid-answer leaves the - // request on the broker rather than having consumed it silently. Asked again by the same - // presenter, an enrolment finishes: the token is held for its key and spent last (issue 083). - _ = delivery.Ack(false) -} - -func (s *Server) reply(ctx context.Context, delivery amqp.Delivery, reply EnrolReply) { - if delivery.ReplyTo == "" { - s.log.Print("an enrolment request named no reply queue, so nothing can be told the answer") - return - } - body, err := json.Marshal(reply) - if err != nil { + if body, err := json.Marshal(reply); err != nil { s.log.Printf("cannot encode a reply: %v", err) - return + } else { + answer, cancel := context.WithTimeout(ctx, 10*time.Second) + if err := m.Answer(answer, body); err != nil { + s.log.Printf("cannot answer an enrolment: %v", err) + } + cancel() } - timeout, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if err := s.channel.PublishWithContext(timeout, "", delivery.ReplyTo, false, false, - amqp.Publishing{ - ContentType: "application/json", - CorrelationId: delivery.CorrelationId, - Body: body, - }); err != nil { - s.log.Printf("cannot reply to %s: %v", delivery.ReplyTo, err) - } + // Settled after the answer is sent, so a controller that dies mid-answer leaves the request on + // the bus rather than having consumed it silently. Asked again by the same presenter, an + // enrolment finishes: the token is held for its key and spent last (issue 083). + _ = m.Took() } -// handleBuilt keeps what a builder said, whichever way it went. +// wasBuilt keeps what a builder said, whichever way it went. // // This is for results nobody was waiting for. A build asked for with `build` is answered directly // to the asker; one triggered any other way is published here, and without this it would be // reported into the void — which is the same as not reporting it. -func (s *Server) handleBuilt(ctx context.Context, delivery amqp.Delivery) { +func (s *Server) wasBuilt(ctx context.Context, m Control) { var result BuildResult - if err := json.Unmarshal(delivery.Body, &result); err != nil { + if err := json.Unmarshal(m.Body(), &result); err != nil { s.log.Printf("a build result could not be read: %v", err) - _ = delivery.Reject(false) + _ = m.Drop() return } if s.recorder == nil { - // Nothing to keep it in. Rejected rather than dropped silently, so the broker's own - // counters show something arriving that nothing handles. + // Nothing to keep it in. Dropped rather than swallowed, so the bus's own counters show + // something arriving that nothing handles. s.log.Printf("a build result arrived and this control plane keeps none") - _ = delivery.Reject(false) + _ = m.Drop() return } // Each build result its own subject: none supersedes another, and recording one twice is // harmless — the build is kept by its id. - subject := "build " + digest(delivery.Body) - s.supersede(subject, delivery) - if err := s.recorder.Built(ctx, result); err != nil { + m.About("build " + digest(m.Body())) + err := s.recorder.Built(ctx, result) + switch s.decide(ctx, m, fmt.Sprintf("a build result from %s", result.On), "", "", err) { + case Hold: // Held while the store cannot take it: a build result lost here is never announced (083). - if s.tryLater(ctx, delivery, subject, fmt.Sprintf("a build result from %s", result.On), err, s.handle) { - return - } + return + case Stale, GiveUp: + _ = m.Drop() + return + } + if err != nil { s.log.Printf("cannot keep a build result from %s: %v", result.On, err) - s.settled(subject, delivery) - _ = delivery.Reject(false) + _ = m.Drop() return } - s.settled(subject, delivery) switch { case result.Failed != "": s.log.Printf("%s could not build %s", result.On, result.Repository) default: s.log.Printf("%s built %s from %s", result.On, result.Repository, result.Commit) } - _ = delivery.Ack(false) + _ = m.Took() } // catchingUp answers a catalogue that has just started and may have missed builds. // -// Acknowledged after the work. A replay that fails for a reason other than the store is not one -// that succeeds by being handed the same request again, so that is acknowledged and said; but a -// store that could not be read right now is held and asked again, bounded, rather than the -// request lost until the catalogue next restarts (issue 083). One request stands for all: a newer -// one supersedes one still held. -func (s *Server) catchingUp(ctx context.Context, delivery amqp.Delivery) { - const subject = "catch-up" - s.supersede(subject, delivery) - holding := false - defer func() { - if !holding { - s.settled(subject, delivery) - _ = delivery.Ack(false) - } - }() +// Settled after the work. A replay that fails for a reason other than the store is not one that +// succeeds by being handed the same request again, so that is settled and said; but a store that +// could not be read right now is held and asked again, bounded, rather than the request lost until +// the catalogue next restarts (issue 083). One request stands for all: a newer one supersedes one +// still held. +func (s *Server) catchingUp(ctx context.Context, m Control) { + m.About("catch-up") if s.replayer == nil { s.log.Printf("a catalogue asked to catch up and this control plane has nothing to replay") + _ = m.Took() return } announcements, err := s.replayer.Announceable(ctx) + switch s.decide(ctx, m, "a catalogue's request to catch up", "", "", err) { + case Hold: + return + case Stale, GiveUp: + _ = m.Took() + return + } if err != nil { - if s.tryLater(ctx, delivery, subject, "a catalogue's request to catch up", err, s.catchingUp) { - holding = true - return - } s.log.Printf("a catalogue asked to catch up and the mesh could not read its builds: %v", err) + _ = m.Took() return } sent := 0 for _, a := range announcements { a.Replay = true - if err := EmitEvent(ctx, s.channel, KeyModuleBuilt, "control-plane", "", a); err != nil { + if err := EmitEvent(ctx, s.bus, KeyModuleBuilt, "control-plane", "", a); err != nil { // Said and abandoned rather than retried: the catalogue asks again every time it // starts, and half a graph delivered twice is no better than half delivered once. s.log.Printf("replaying %s at %s failed, and the rest is abandoned: %v", a.Module, short(a.Commit), err) + _ = m.Took() return } sent++ } s.log.Printf("a catalogue asked to catch up; re-announced %d build(s)", sent) + _ = m.Took() } -// upgraded hands one announcement to whatever is following them. +// moved hands one announcement to whatever is following them. // -// **Acknowledged whatever happens, but one thing.** A failure here is usually the control plane -// being unable to act on an upgrade — a machine that cannot be resolved, a broker that will not -// take a declaration — and none of those get better by being handed the same message again. The -// one exception is the upgrader saying the store could not be read for the moment (ErrTryAgain): -// that is held and asked again, bounded (novox/hq issue 083). Only the upgrader's word counts -// here, not an error that merely looks like an outage — a push that timed out on the second -// machine is not asked again, or the first would be pushed every few seconds for two minutes. -// A newer move of the same module supersedes one still held. -func (s *Server) upgraded(ctx context.Context, delivery amqp.Delivery) { +// **Settled whatever happens, but one thing.** A failure here is usually the controller being +// unable to act on an upgrade — a machine that cannot be resolved, a bus that will not take a +// declaration — and none of those get better by being handed the same message again. The one +// exception is the upgrader saying the store could not be read for the moment (ErrTryAgain): that +// is held and asked again, bounded (novox/hq issue 083). Only the upgrader's word counts here, not +// an error that merely looks like an outage — a push that timed out on the second machine is not +// asked again, or the first would be pushed every few seconds for two minutes. A newer move of the +// same module supersedes one still held. +func (s *Server) moved(ctx context.Context, m Control) { var u Upgraded - _ = json.Unmarshal(delivery.Body, &u) - subject := "upgrade " + u.Module - s.supersede(subject, delivery) - holding := false - defer func() { - if !holding { - s.settled(subject, delivery) - _ = delivery.Ack(false) - } - }() - if err := json.Unmarshal(delivery.Body, &u); err != nil { + if err := json.Unmarshal(m.Body(), &u); err != nil { s.log.Printf("an upgrade announcement could not be read: %v", err) + _ = m.Took() return } + m.About("upgrade " + u.Module) if u.Module == "" { s.log.Printf("an upgrade announcement named no module; ignored") + _ = m.Took() return } - if err := s.upgrader.Upgraded(ctx, u); err != nil { - // Shutting down is not an answer about the announcement: left for the broker. - if ctx.Err() != nil { - holding = true - return - } - if errors.Is(err, ErrTryAgain) && - s.tryLater(ctx, delivery, subject, fmt.Sprintf("%s's move to %s", u.Module, short(u.Commit)), err, s.upgraded) { - holding = true - return - } + err := s.upgrader.Upgraded(ctx, u) + // Only "not now" is worth holding. Anything else is an answer, and the window would read a + // timed-out push as an outage and ask for it again. + holdable := err + if !errors.Is(err, ErrTryAgain) { + holdable = nil + } + what := fmt.Sprintf("%s's move to %s", u.Module, short(u.Commit)) + switch s.decide(ctx, m, what, "", "", holdable) { + case Hold: + return + case Stale, GiveUp: + _ = m.Took() + return + } + if err != nil { s.log.Printf("%s moved to %s and the mesh could not act on it: %v", u.Module, short(u.Commit), err) } + _ = m.Took() +} + +// digest names a message by its content. +func digest(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) } // short is a commit as people read it. diff --git a/internal/link/stale_report_test.go b/internal/link/stale_report_test.go new file mode 100644 index 0000000..2e0b949 --- /dev/null +++ b/internal/link/stale_report_test.go @@ -0,0 +1,135 @@ +package link + +import ( + "context" + "errors" + "testing" +) + +// Supersession as a check rather than a memory (design 25 §3). +// +// Holding a message in memory let the controller drop an older report when a newer one for the +// same node arrived. On the bus being built the message belongs to the server and comes back +// whatever happened meanwhile — so the older report is redelivered *after* the newer was applied, +// and acting on it would undo the newer. +// +// The answer was already in the message: a report carries the digest of the declaration it is +// about, so "is this the past?" is a question the message answers. + +// sentAndHeard records reports and knows what was last sent, which is the pair the check needs. +type sentAndHeard struct { + sent string + heard []Report + err error +} + +func (s *sentAndHeard) Heard(_ context.Context, r Report) error { + if s.err != nil { + return s.err + } + s.heard = append(s.heard, r) + return nil +} + +func (s *sentAndHeard) Outstanding(context.Context, string) (string, error) { return s.sent, nil } + +// A report about a declaration the mesh has moved past is settled and not acted on. Settled rather +// than dropped, because there is nothing wrong with the message — it is simply the past, and +// redelivering it for ever is worse than letting it go. +func TestAReportAboutASupersededDeclarationIsNotActedOn(t *testing.T) { + store := &sentAndHeard{sent: "d2"} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) + + if len(store.heard) != 0 { + t.Fatalf("a report about a superseded declaration was acted on: %+v", store.heard) + } + if !to.acked { + t.Fatalf("a superseded report was not settled, so it comes back for ever: %+v", *to) + } +} + +// The report about the declaration that *is* outstanding is acted on, and so is one from a node +// the mesh has no digest for — an older host that says nothing about which declaration it applied +// has nothing to be judged against, and refusing it would silence every node built before reports +// carried the digest. +func TestAReportAboutTheOutstandingDeclarationIsActedOn(t *testing.T) { + for _, c := range []struct{ what, sent, declared string }{ + {"the one outstanding", "d2", "d2"}, + {"a report that says nothing about which", "d2", ""}, + {"a node nothing was ever sent", "", "d1"}, + } { + store := &sentAndHeard{sent: c.sent} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", c.declared))) + if len(store.heard) != 1 || !to.acked { + t.Errorf("%s: was not acted on and acknowledged: heard %+v, settled %+v", + c.what, store.heard, *to) + } + } +} + +// **Staleness is decided before the store is waited on**, not after: a redelivery that lost its +// race is not worth holding a slot in the window that a current message needs. +func TestASupersededReportIsNotHeldForTheStore(t *testing.T) { + store := &sentAndHeard{sent: "d2", err: errors.Join(ErrTryAgain, errors.New("starting up"))} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) + if !to.acked || len(in.held) != 0 { + t.Fatalf("a superseded report waited for the store: %+v, %d held", *to, len(in.held)) + } +} + +// **Half of a report is not about a declaration, and that half is never stale.** +// +// What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted +// node found and is keeping, a node moving its overlay key — reaches the mesh on a report and +// nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is +// coming, because the node said it once. So a report carrying any of these is acted on whenever it +// arrives, however far the mesh has moved on. +func TestAReportCarryingWhatOnlyTheNodeKnowsIsActedOnHoweverOldItIs(t *testing.T) { + for _, c := range []struct { + what string + report Report + }{ + {"a rekey", Report{Node: "anchor", Declared: "d1", + Rekey: &Rekey{Previous: "k1", OverlayKey: "k2"}}}, + {"the tunnel it carried", Report{Node: "anchor", Declared: "d1", + Tunnel: &CarriedTunnel{Interface: "wg0", State: "taken"}}}, + {"what an adopted node holds", Report{Node: "anchor", Declared: "d1", + Held: []Held{{ID: "conf", Module: "web", Kind: "file"}}}}, + {"the firewall it found", Report{Node: "anchor", Declared: "d1", Firewall: "ufw"}}, + {"what is reachable on it", Report{Node: "anchor", Declared: "d1", + Reachable: []Reach{{Protocol: "tcp", Port: 443}}}}, + {"the ports its own bundle holds", Report{Node: "anchor", Declared: "d1", + Carried: []int{5432}}}, + } { + store := &sentAndHeard{sent: "d9"} + s, in := serving() + s.listener = store + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindReport, c.report)) + if len(store.heard) != 1 { + t.Errorf("%s was set aside as stale, and the mesh will never hear it again: %+v", + c.what, *to) + } + } +} + +// A heartbeat is not held for the store: the next one is a minute away, and one kept for two +// minutes to be written late says nothing the one after it will not say better. +func TestAHeartbeatIsNotHeldForTheStore(t *testing.T) { + s, in := serving() + s.listener = heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))} + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindHeartbeat, Alive{Node: "anchor"})) + if !to.acked || len(in.held) != 0 { + t.Fatalf("a heartbeat was held for the store: %+v, %d held", *to, len(in.held)) + } +} diff --git a/internal/link/store_window_test.go b/internal/link/store_window_test.go index f0fcd2d..80b6a4b 100644 --- a/internal/link/store_window_test.go +++ b/internal/link/store_window_test.go @@ -2,15 +2,11 @@ package link import ( "context" - "encoding/json" "errors" "fmt" - "io" - "log" "testing" "github.com/jackc/pgx/v5/pgconn" - amqp "github.com/rabbitmq/amqp091-go" ) // What a store restarting under an adoption answers with (novox/hq issues 082, 083). @@ -28,29 +24,6 @@ type replaysWith struct{ err error } func (r replaysWith) Announceable(context.Context) ([]Announcement, error) { return nil, r.err } -type settledAs struct{ acked, nacked, requeued, rejected bool } - -func (a *settledAs) Ack(uint64, bool) error { a.acked = true; return nil } -func (a *settledAs) Nack(_ uint64, _ bool, requeue bool) error { - a.nacked, a.requeued = true, requeue - return nil -} -func (a *settledAs) Reject(uint64, bool) error { a.rejected = true; return nil } - -func a(t *testing.T, to *settledAs, key string, v any) amqp.Delivery { - t.Helper() - body, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - tag++ - return amqp.Delivery{Acknowledger: to, RoutingKey: key, Body: body, DeliveryTag: tag} -} - -func quietServer() *Server { return &Server{log: log.New(io.Discard, "", 0)} } - -func (a *settledAs) held() bool { return !a.acked && !a.nacked && !a.rejected } - // A build result the store could not take right now is handed back; one it refused is rejected, // as before; one it kept is acknowledged. func TestABuildResultWaitsOutARestartingStore(t *testing.T) { @@ -58,16 +31,16 @@ func TestABuildResultWaitsOutARestartingStore(t *testing.T) { for _, c := range []struct { what string err error - want func(*settledAs) bool + want func(*settled) bool }{ - {"restarting", restarting, func(s *settledAs) bool { return s.held() }}, - {"refused", errors.New("no such module"), func(s *settledAs) bool { return s.rejected && !s.nacked }}, - {"kept", nil, func(s *settledAs) bool { return s.acked && !s.nacked }}, + {"restarting", restarting, func(s *settled) bool { return s.unsettled() }}, + {"refused", errors.New("no such module"), func(s *settled) bool { return s.rejected && !s.nacked }}, + {"kept", nil, func(s *settled) bool { return s.acked && !s.nacked }}, } { - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: c.err} - to := &settledAs{} - s.handleBuilt(context.Background(), a(t, to, KeyBuilt, built)) + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindBuilt, built)) if !c.want(to) { t.Errorf("%s: a build result was settled as %+v", c.what, *to) } @@ -81,17 +54,17 @@ func TestAnUpgradeWaitsOutARestartingStoreAndNothingElse(t *testing.T) { for _, c := range []struct { what string err error - want func(*settledAs) bool + want func(*settled) bool }{ - {"the store away, said by the upgrader", errors.Join(ErrTryAgain, restarting), func(s *settledAs) bool { return s.held() }}, - {"a push that timed out", context.DeadlineExceeded, func(s *settledAs) bool { return s.acked && !s.nacked }}, - {"cannot act", errors.New("anchor cannot be resolved"), func(s *settledAs) bool { return s.acked && !s.nacked }}, - {"acted", nil, func(s *settledAs) bool { return s.acked && !s.nacked }}, + {"the store away, said by the upgrader", errors.Join(ErrTryAgain, restarting), func(s *settled) bool { return s.unsettled() }}, + {"a push that timed out", context.DeadlineExceeded, func(s *settled) bool { return s.acked && !s.nacked }}, + {"cannot act", errors.New("anchor cannot be resolved"), func(s *settled) bool { return s.acked && !s.nacked }}, + {"acted", nil, func(s *settled) bool { return s.acked && !s.nacked }}, } { - s := quietServer() + s, in := serving() s.upgrader = upgradesWith{err: c.err} - to := &settledAs{} - s.upgraded(context.Background(), a(t, to, "upgraded", moved)) + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindModuleMoved, moved)) if !c.want(to) { t.Errorf("%s: an upgrade was settled as %+v", c.what, *to) } @@ -104,16 +77,16 @@ func TestACatchUpWaitsOutARestartingStore(t *testing.T) { for _, c := range []struct { what string err error - want func(*settledAs) bool + want func(*settled) bool }{ - {"restarting", restarting, func(s *settledAs) bool { return s.held() }}, - {"unreadable", errors.New("a build row is malformed"), func(s *settledAs) bool { return s.acked && !s.nacked }}, - {"nothing to replay", nil, func(s *settledAs) bool { return s.acked && !s.nacked }}, + {"restarting", restarting, func(s *settled) bool { return s.unsettled() }}, + {"unreadable", errors.New("a build row is malformed"), func(s *settled) bool { return s.acked && !s.nacked }}, + {"nothing to replay", nil, func(s *settled) bool { return s.acked && !s.nacked }}, } { - s := quietServer() + s, in := serving() s.replayer = replaysWith{err: c.err} - to := &settledAs{} - s.catchingUp(context.Background(), a(t, to, "catch-up", map[string]string{})) + to := &settled{} + s.act(context.Background(), in.sends(t, to, KindCatchUp, map[string]string{})) if !c.want(to) { t.Errorf("%s: a catch-up request was settled as %+v", c.what, *to) } @@ -121,15 +94,15 @@ func TestACatchUpWaitsOutARestartingStore(t *testing.T) { } // Shutting down is not an answer about a message: one handled with a cancelled context is left -// unsettled, for the broker to hand to whatever consumes next (issue 083, review). -func TestAMessageHandledDuringShutdownIsLeftForTheBroker(t *testing.T) { +// unsettled, for the bus to hand to whatever consumes next (issue 083, review). +func TestAMessageHandledDuringShutdownIsLeftForTheBus(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: context.Canceled} - to := &settledAs{} - s.handleBuilt(ctx, a(t, to, KeyBuilt, BuildResult{On: "anchor", Repository: "/r", Commit: "abc"})) - if !to.held() { + to := &settled{} + s.act(ctx, in.sends(t, to, KindBuilt, BuildResult{On: "anchor", Repository: "/r", Commit: "abc"})) + if !to.unsettled() { t.Fatalf("a build result handled during shutdown was settled, and so lost: %+v", *to) } } @@ -137,45 +110,45 @@ func TestAMessageHandledDuringShutdownIsLeftForTheBroker(t *testing.T) { // Two identical build results: the newer sets the older aside rather than leaving it unsettled // for ever, holding a place in the prefetch. func TestAnIdenticalBuildResultSetsTheHeldOneAside(t *testing.T) { - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: restarting} built := BuildResult{On: "anchor", Repository: "/r", Commit: "abc"} - first, second := &settledAs{}, &settledAs{} - s.handleBuilt(context.Background(), a(t, first, KeyBuilt, built)) - s.handleBuilt(context.Background(), a(t, second, KeyBuilt, built)) - if !first.acked || !second.held() || len(s.parked) != 1 { + first, second := &settled{}, &settled{} + s.act(context.Background(), in.sends(t, first, KindBuilt, built)) + s.act(context.Background(), in.sends(t, second, KindBuilt, built)) + if !first.acked || !second.unsettled() || len(in.held) != 1 { t.Fatalf("an identical build result did not set the held one aside: first %+v second %+v, %d held", - *first, *second, len(s.parked)) + *first, *second, len(in.held)) } } // What is held stops short of the prefetch, so the loop always has room to answer an enrolment. func TestWhatIsHeldLeavesRoomInThePrefetch(t *testing.T) { - s := quietServer() + s, in := serving() s.recorder = recordsWith{err: restarting} - var last *settledAs + var last *settled for i := 0; i < Prefetch; i++ { - last = &settledAs{} - s.handleBuilt(context.Background(), a(t, last, KeyBuilt, BuildResult{On: "anchor", Commit: fmt.Sprint(i)})) + last = &settled{} + s.act(context.Background(), in.sends(t, last, KindBuilt, BuildResult{On: "anchor", Commit: fmt.Sprint(i)})) } - if len(s.parked) != Prefetch-PrefetchHeadroom { - t.Fatalf("%d messages were held; the ceiling is %d", len(s.parked), Prefetch-PrefetchHeadroom) + if len(in.held) != Prefetch-PrefetchHeadroom { + t.Fatalf("%d messages were held; the ceiling is %d", len(in.held), Prefetch-PrefetchHeadroom) } - if last.held() { + if last.unsettled() { t.Fatalf("a message past the ceiling was held: %+v", *last) } } -// An upgrade handled during shutdown is left for the broker too — the upgrader's error is the +// An upgrade handled during shutdown is left for the bus too — the upgrader's error is the // cancelled context, which is no answer about the announcement. -func TestAnUpgradeHandledDuringShutdownIsLeftForTheBroker(t *testing.T) { +func TestAnUpgradeHandledDuringShutdownIsLeftForTheBus(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s := quietServer() + s, in := serving() s.upgrader = upgradesWith{err: context.Canceled} - to := &settledAs{} - s.upgraded(ctx, a(t, to, "upgraded", Upgraded{Module: "gitea", Commit: "abcdef0123"})) - if !to.held() { - t.Fatalf("an upgrade handled during shutdown was settled, and so lost: %+v", *to) + to := &settled{} + s.act(ctx, in.sends(t, to, KindModuleMoved, Upgraded{Module: "gitea", Commit: "abcdef0123"})) + if !to.unsettled() { + t.Fatalf("an upgrade was settled during shutdown, and so lost: %+v", *to) } } diff --git a/internal/link/window.go b/internal/link/window.go new file mode 100644 index 0000000..91db759 --- /dev/null +++ b/internal/link/window.go @@ -0,0 +1,120 @@ +package link + +import ( + "errors" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// The store window, as a decision rather than a mechanism. +// +// The guarantee (novox/hq ADR 0083): a push the controller cannot record because its store is +// restarting is **held and retried**, not dropped and not falsely acknowledged. On the bus the +// mesh runs on today that is done by keeping the delivery unacknowledged in memory and settling +// it later. On the bus being built it is a `nak` with a delay: the server holds it and redelivers, +// so the controller keeps no list of parked messages and a controller that restarts mid-window +// loses nothing it was holding. +// +// **The decision is the same either way, and the mechanism is not the interesting part.** What is +// interesting is that moving the holding into the server introduces a problem the in-memory +// version did not have, and the answer was already in the message. + +// Verdict is what to do with one control message. +type Verdict int + +const ( + // Take it: apply, then acknowledge. + Take Verdict = iota + // Hold it: the store cannot record this yet. Nak with a delay and let the server redeliver. + Hold + // Stale: this is about a declaration the node has already moved past, and applying it would + // undo what came after. Acknowledge without acting — redelivering forever is worse. + Stale + // GiveUp: the store has not come back in time. Settle it and say so, loudly. + GiveUp +) + +// StoreWindow decides. Pure, so the guarantee is testable without a bus, a store or a clock. +type StoreWindow struct { + // GiveUpAfter is how long one message may be held before it is let go with a line saying so. + GiveUpAfter time.Duration +} + +// Decide answers for one delivery. +// +// - err is what the store said, or nil. +// - declaredIn is the digest of the declaration this message is about, empty when it is not +// about one (an enrolment, a build result). +// - outstanding is the digest the mesh last sent that node, empty when it has sent none. +// - heldFor is how long this message has already been held; zero on first delivery. +func (w StoreWindow) Decide(err error, declaredIn, outstanding string, heldFor time.Duration) Verdict { + // **Staleness is checked before the store, not after.** A redelivery that lost its race is + // not worth waiting on a store for, and asking the store first would mean a message about a + // superseded declaration holding a slot in the window that a current one needs. + if Superseded(declaredIn, outstanding) { + return Stale + } + if err == nil { + return Take + } + if !errors.Is(err, ErrTryAgain) && !inventory.Unreachable(err) { + // Not the store being away: a refusal is an answer, and holding it would turn a message + // the mesh understood into one it retries forever. + return Take + } + if heldFor >= w.GiveUpAfter { + return GiveUp + } + return Hold +} + +// Superseded says a message is about a declaration the mesh has already moved past. +// +// Stated on its own because it is asked in two places for one reason: here, so the window's whole +// decision is in one pure function, and by the serving loop *before* it asks the store, because +// that is the point — a message about the past must not wait on a store, or it holds a slot in the +// window that a current message needs. +// +// Unanswerable is not stale. A message that names no declaration, and a node the mesh has never +// sent one, both give nothing to compare: the mesh acts on the message rather than guessing, which +// is also what keeps a host built before reports carried the digest from going silent. +func Superseded(declaredIn, outstanding string) bool { + return declaredIn != "" && outstanding != "" && declaredIn != outstanding +} + +// RedeliverAfter is how long the server should hold a naked message before trying again. +// +// Backed off, and bounded. A store restarting is back in seconds; a store that is gone is not +// helped by being asked every second, and the delay is what keeps a window of held messages from +// becoming a spin. +func RedeliverAfter(heldFor time.Duration) time.Duration { + switch { + case heldFor < 5*time.Second: + return time.Second + case heldFor < 30*time.Second: + return 5 * time.Second + default: + return 15 * time.Second + } +} + +// The problem holding-in-the-server introduces, and why the answer was already in the message. +// +// Holding a delivery in memory let the controller do something a server cannot: when a newer +// report for the same node arrived, it dropped the older one, "because acting on it after the +// newer would undo the newer". A `nak`ed message is the server's, and the server will redeliver +// it whatever else has happened in the meantime — so the older report comes back *after* the +// newer was applied, and applying it would undo exactly what that comment describes. +// +// **A report already says which declaration it is about.** `Declared` is the digest of the exact +// bytes the mesh sent, and it exists because an earlier attempt to order reports by time lost the +// race it invited — an apply that started under the previous declaration finishes after the next +// is sent, and the report reads as newer than the send. Clocks cannot answer *which*; the digest +// is the answer itself. +// +// So supersession stops being a thing the controller remembers and becomes a thing it checks: a +// report whose digest is not the one outstanding for that node is stale, and is acknowledged +// without being acted on. Which is the same shape as a node refusing a superseded declaration by +// sequence (novox/hq issue 107) — ordering settled by what the message says, not by when it +// happened to arrive. diff --git a/internal/link/window_test.go b/internal/link/window_test.go new file mode 100644 index 0000000..531efea --- /dev/null +++ b/internal/link/window_test.go @@ -0,0 +1,84 @@ +package link + +import ( + "errors" + "testing" + "time" +) + +func window() StoreWindow { return StoreWindow{GiveUpAfter: 2 * time.Minute} } + +// The guarantee itself (novox/hq ADR 0083): a push the store cannot record is held, not dropped +// and not falsely acknowledged. +func TestAMessageTheStoreCannotTakeYetIsHeld(t *testing.T) { + if got := window().Decide(ErrTryAgain, "", "", 0); got != Hold { + t.Fatalf("got %v; a push the store could not record was not held", got) + } +} + +// A refusal is an answer. Holding it would turn a message the mesh understood into one it +// retries forever. +func TestARefusalIsNotHeld(t *testing.T) { + if got := window().Decide(errors.New("that node does not exist"), "", "", 0); got != Take { + t.Fatalf("got %v; a refusal was mistaken for the store being away", got) + } +} + +// Held has a limit, and past it the message is settled rather than held for ever. +func TestAStoreThatNeverComesBackEndsTheHold(t *testing.T) { + if got := window().Decide(ErrTryAgain, "", "", 3*time.Minute); got != GiveUp { + t.Fatalf("got %v; the window has no end", got) + } + if got := window().Decide(ErrTryAgain, "", "", time.Minute); got != Hold { + t.Fatalf("got %v; the window ended early", got) + } +} + +// **The problem that holding in the server introduces.** A naked message is redelivered whatever +// else happened meanwhile, so a report about a superseded declaration comes back after the newer +// one was applied — and applying it would undo the newer. +func TestAReportAboutASupersededDeclarationIsNotApplied(t *testing.T) { + got := window().Decide(nil, "digest-of-the-old-one", "digest-of-the-current-one", 0) + if got != Stale { + t.Fatalf("got %v; a redelivery that lost its race would have undone what came after", got) + } +} + +func TestAReportAboutTheOutstandingDeclarationIsApplied(t *testing.T) { + if got := window().Decide(nil, "same", "same", 0); got != Take { + t.Fatalf("got %v; a current report was discarded", got) + } +} + +// A message that is about no declaration — an enrolment, a build result — is never stale: there +// is nothing for it to be out of date with. +func TestAMessageAboutNoDeclarationIsNeverStale(t *testing.T) { + if got := window().Decide(nil, "", "whatever-is-outstanding", 0); got != Take { + t.Fatalf("got %v; an enrolment was treated as a stale report", got) + } + if got := window().Decide(nil, "a-digest", "", 0); got != Take { + t.Fatalf("got %v; a report was called stale against a node that was sent nothing", got) + } +} + +// Staleness is decided before the store is waited on: a redelivery that lost its race must not +// hold a slot in the window that a current message needs. +func TestAStaleMessageIsNotHeldForTheStore(t *testing.T) { + if got := window().Decide(ErrTryAgain, "old", "current", 0); got != Stale { + t.Fatalf("got %v; a superseded message was held for a store it would never be applied to", got) + } +} + +// The delay backs off: a store that is gone is not helped by being asked every second, and the +// delay is what keeps a window of held messages from becoming a spin. +func TestRedeliveryBacksOff(t *testing.T) { + first := RedeliverAfter(0) + later := RedeliverAfter(10 * time.Second) + last := RedeliverAfter(time.Minute) + if !(first < later && later < last) { + t.Fatalf("delays do not back off: %v %v %v", first, later, last) + } + if last > 30*time.Second { + t.Fatalf("a held message waits %v between attempts, which is longer than a store restart", last) + } +}