diff --git a/examples/README.md b/examples/README.md index 5ad1d1c..a9691ee 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,7 +12,8 @@ mesh runs on: 2 a database per context `inventory`, `identity` and `licences` 3 those contexts' schemas mesh-controller migrate 4 the bus's certificate mesh-controller broker certificate -5 the bus runs NATS, with the controller's account as its first user +5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it + (ADR 0333), with the controller's account as its first user 6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS) ``` @@ -26,6 +27,12 @@ rootsecrets.go). A template whose bus connection it does not recognise is refuse naming the placeholder it looked for. The bundle that raised the previous broker was deleted with ADR 0331; there is one bundle, and every lab bed names it. +**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container +`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in +`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus +port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once, +on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest. + Build an installer carrying a builder, and point it at this: ``` diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 289e3e2..2e32f68 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -141,6 +141,30 @@ "migrate"], "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] }, + // **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container + // `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in + // /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the + // catalogue's module declares. So adopting it as that module at the broker step keeps it: the + // module's built image takes the place of the upstream one below, once, on the same streams, users + // and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other + // against the module's own manifest (busLayout), at the broker step and in its tests. + // + // Directories on the machine, not volumes the runtime names: the module mounts these paths, and the + // control plane, once it runs as the mesh's process, reads the certificate at its path on the machine. + // The certificate's directory is readable by everyone because the certificate is public and the control + // plane runs as nobody; its key is written 0600 by the program that makes it. + { + "id": "bus-tls-dir", + "type": "directory", + "path": "/var/lib/mesh-broker-tls", + "mode": "0755" + }, + { + "id": "bus-data-dir", + "type": "directory", + "path": "/var/lib/mesh-broker-nats", + "mode": "0700" + }, { "id": "bus-certificate", "type": "action", @@ -151,42 +175,53 @@ // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh // to ask an authority of. - // `--user 0:0` because the volume is root's and this image runs as nobody, which is right - // for the long-running control plane and wrong for a one-shot writing into a fresh volume. - "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", + // `--user 0:0` because the directory is root's and this image runs as nobody, which is right + // for the long-running control plane and wrong for a one-shot writing into a fresh directory. + "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "broker", "certificate", "--into", "/tls"], - "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", + "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "broker", "certificate", "--check", "--into", "/tls"] }, + // **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are + // the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer + // run again after the adoption applies this bundle again, and an ordinary file would put the + // bundle's one user back over every user the mesh composed, and leave the node unable to reach its + // bus. Until the adoption the installer places the composed list over the seed (enrol.go). { "id": "bus-conf-dir", "type": "directory", - "path": "/var/lib/mesh-bus-conf", + "path": "/var/lib/nats-module/conf", "mode": "0700" }, { "id": "bus-conf", "type": "file", - "path": "/var/lib/mesh-bus-conf/nats.conf", + "path": "/var/lib/nats-module/conf/nats.conf", "mode": "0644", - "content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" + "create-once": true, + "content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" }, { "id": "bus-accounts", "type": "file", - "path": "/var/lib/mesh-bus-conf/accounts.conf", + "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0600", + "create-once": true, "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, + // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the + // mesh, takes its place there once, on these same directories (ADR 0333 §3). The published port is the + // node's bus port, 5671 unless given, which every enrolment token names; the installer gives the + // module the setting that maps its declared 4222 to it (ADR 0333 §2). { "id": "broker", "type": "container", - "name": "mesh-broker", + "name": "mesh-broker-nats", "image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927", "ports": ["5671:4222", "127.0.0.1:8222:8222"], - "volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"], + "volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"], "args": ["-c", "/etc/nats/nats.conf", "-js"] }, { @@ -202,7 +237,7 @@ "image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "network": "host", "args": ["serve"], - "volumes": ["mesh-broker-tls:/broker-tls:ro"], + "volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"], "env": { "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 21a1dee..c047eb1 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -734,7 +734,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- broker --------------------------------------------------------------------------- say("broker — the foundation's bus, adopted as the nats module: one bus, not two") - if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { + if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, d.Fetch, say); err != nil { return result, failed(StepBroker, err) } diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index b0d13d2..89e1228 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -94,10 +94,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla switch mine, err := identity.Load(where); { case err == nil && mine.Node == o.Node: say(" already enrolled " + o.Node + " — " + where) - // **Enrolled is not adopted** (novox/hq ADR 0331 §5). This run's apply wrote the bundle's - // user list again — the controller's entry and nobody else's — over the list placed when - // the machine enrolled. Until the bus is the nats module, that list is still the - // installer's to place, so it is placed again, before anything dials the bus as a node. + // **Enrolled is not adopted** (novox/hq ADR 0331 §5). Until the bus is the nats module, its + // user list is the installer's to place, so it is placed again, before anything dials the + // bus as a node: the controller's composition is the one that is right, whatever an earlier + // run left there. The bundle seeds the list only where none is (create-once, ADR 0333 §4). if err := placeUntilAdopted(ctx, control, say); err != nil { return out, err } @@ -168,9 +168,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla // configuration includes, and the container that reads it. The installer raised them, so it is the // one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the // nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5). +// +// **The module's path and the module's container** (novox/hq ADR 0333 §4): the bundle raises the bus +// where the `nats` module keeps it, so the list placed here is the one the module goes on writing, in +// the same file, read by the same container. The tests hold both to the bundle and to the module's +// manifest (`bus-users` and its server container). const ( - busAccounts = "/var/lib/mesh-bus-conf/accounts.conf" - busContainer = "mesh-broker" + busAccounts = "/var/lib/nats-module/conf/accounts.conf" + busContainer = "mesh-broker-nats" ) // placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go index 111d000..031b823 100644 --- a/internal/bootstrap/enrol_test.go +++ b/internal/bootstrap/enrol_test.go @@ -315,8 +315,8 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) { placed, enrol := []int{}, -1 for i, c := range runtime.commands { - if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") && - strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") { + if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker-nats") && + strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") { placed = append(placed, i) } if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") { @@ -365,7 +365,7 @@ func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *tes placed := false for _, c := range runtime.commands { if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") && - strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker") { + strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker-nats") { placed = true } } diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index c8aadbc..216bc88 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "net/http" "strings" + "time" "github.com/novox/mesh-host/internal/declaration" ) @@ -165,13 +167,21 @@ const BusModule = "nats" // (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the // bridge to here and is not run again. // -// **What is compared is the container's name, not its image.** The store's module pins the upstream -// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's -// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is -// not the foundation's container by name would raise a second bus beside the first, and is refused -// by name before anything is registered. +// **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container +// name, the same streams directory, configuration directory, user list and certificate. That is checked +// here against the module's own manifest before anything is registered, and a difference is refused +// naming each one — under another name or on other paths the module would raise a second bus beside the +// first, or an empty one in its place. +// +// **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream +// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot +// tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on +// the module's image, on the same streams, users and certificate. This says so before the push and waits +// after it until the container runs the module's image and the bus answers, so nothing after this step +// dials a bus that is restarting. func InstallBroker(ctx context.Context, o Options, control controlPlane, - foundation *declaration.Declaration, say func(string)) error { + foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error), + say func(string)) error { const module = BusModule manifest, err := readManifest(o.Catalogue, module) @@ -182,12 +192,23 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, if err != nil { return err } - if err := busServerIsTheFoundations(manifest, bus, module); err != nil { + if err := busLayoutIsTheModules(manifest, foundation, module); err != nil { return err } say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept") if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)", - nil, say); err != nil { + func() error { + if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image { + say(" already restarted " + bus.Name + " — it runs the module's image") + return nil + } + say(" restarting " + bus.Name + " — once, onto the " + module + " module's image; its " + + "streams, users and certificate stay where they are (novox/hq ADR 0333 §3)") + return nil + }, say); err != nil { + return err + } + if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil { return err } say(" adopted " + bus.Name + " — the foundation's bus is now the " + module + @@ -195,41 +216,190 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, return nil } -// busServerIsTheFoundations checks that the bus module's server container is the one the -// foundation raised, by name, so the applier takes it over rather than raising a second bus. -func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error { +// busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own +// loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only +// once the server is up and JetStream has its streams back. +const busMonitor = "http://127.0.0.1:8222/healthz" + +// runningImage is the image the named container was created from, or "" when there is none. +func runningImage(ctx context.Context, control controlPlane, name string) string { + asking, cancel := context.WithTimeout(ctx, control.timeout) + defer cancel() + out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name) + if err != nil { + return "" + } + return strings.TrimSpace(out) +} + +// waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised +// it from — the module's, after the applier's one recreate — and the bus answers on its monitor. +// +// Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is +// refused, naming what it runs: the push was sent and the node has not applied it, and every step after +// this one reaches the mesh through that bus. +func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container, + fetch func(context.Context, string) (int, string, error), say func(string)) error { + + deadline := time.Now().Add(o.Wait) + var running, said string + for { + running = runningImage(ctx, control, bus.Name) + if running != "" && running != bus.Image { + code, body, err := fetch(ctx, busMonitor) + if err == nil && code == http.StatusOK { + say(" bus answers " + bus.Name + " on " + running) + return nil + } + said = strings.TrimSpace(body) + if err != nil { + said = err.Error() + } + } + if !time.Now().Before(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + switch { + case running == "": + return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+ + "after this step could reach the mesh. The node-engine's journal says why it did not raise it", + bus.Name, BusModule) + case running == bus.Image: + return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+ + "has not applied it. Its streams are untouched; run this installer again once the node-engine "+ + "applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule) + } + return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s", + bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said)) +} + +// BusUsersID is the bundle's resource for the bus's first user list. +const BusUsersID = "bus-accounts" + +// busLayout is where a bus lives: its container, the machine's directories it mounts as its streams, +// its configuration and its certificate, and the file its user list is. +type busLayout struct { + Container, Streams, Conf, Certificate, Users string +} + +// Where the server reads each, inside its container: the same in the bundle and the module, since both +// run the same server with the same configuration. +const ( + busStreamsInside = "/data" + busConfInside = "/etc/nats" + busCertificateInside = "/tls" +) + +// foundationBusLayout reads the layout of the bus the bundle raises. +func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) { + bus, err := brokerIn(foundation) + if err != nil { + return busLayout{}, err + } + at := mountedFrom(bus.Volumes) + l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside], + Certificate: at[busCertificateInside]} + for _, r := range foundation.Resources { + if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID { + l.Users = f.Path + } + } + return l, nil +} + +// moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of +// that container with every `${access:}` resolved to the access's path, and its `bus-users`. +func moduleBusLayout(manifest []byte, module string) (busLayout, error) { var m struct { + BusUsers string `json:"bus-users"` Resources []struct { - Type string `json:"type"` - Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Volumes []string `json:"volumes"` } `json:"resources"` + Accesses []struct { + ID string `json:"id"` + Path string `json:"path"` + } `json:"accesses"` } if err := json.Unmarshal(manifest, &m); err != nil { - return fmt.Errorf("the %s module's manifest is not readable: %w", module, err) + return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err) } - var names []string - for _, r := range m.Resources { - if r.Type != "container" { - continue + var containers []int + for i, r := range m.Resources { + if r.Type == "container" { + containers = append(containers, i) } - if r.Name == bus.Name { - return nil + } + if len(containers) != 1 { + var names []string + for _, i := range containers { + names = append(names, m.Resources[i].Name) } - names = append(names, r.Name) + return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+ + "declares one container", module, describeNames(names)) + } + server := m.Resources[containers[0]] + resolved := make([]string, 0, len(server.Volumes)) + for _, v := range server.Volumes { + for _, a := range m.Accesses { + v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path) + } + resolved = append(resolved, v) + } + at := mountedFrom(resolved) + return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside], + Certificate: at[busCertificateInside], Users: m.BusUsers}, nil +} + +// busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by +// field, so the applier takes it over rather than raising a second bus beside it or an empty one in its +// place (novox/hq ADR 0333 §1). Every difference is named, with both sides. +func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error { + theirs, err := moduleBusLayout(manifest, module) + if err != nil { + return err + } + ours, err := foundationBusLayout(foundation) + if err != nil { + return err + } + var differ []string + for _, f := range []struct{ what, module, bundle string }{ + {"the server container", theirs.Container, ours.Container}, + {"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams}, + {"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf}, + {"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate}, + {"its user list", theirs.Users, ours.Users}, + } { + if f.module == "" || f.module != f.bundle { + differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what, + f.module, f.bundle)) + } + } + if len(differ) == 0 { + return nil } return fmt.Errorf( - "the %s module's server is %s, and the foundation's bus is the container %q.\n"+ - "The module adopts the foundation's bus in place, so its server has to carry the name the "+ - "foundation raised; under another name it would raise a second bus beside the first, and "+ - "the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered", - module, describeNames(names), bus.Name) + "the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+ + "The module adopts the foundation's bus in place, so the bundle raises it with the module's "+ + "name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+ + "one on other paths (novox/hq ADR 0333 §1). Nothing has been registered", + module, strings.Join(differ, "\n - ")) } func describeNames(names []string) string { if len(names) == 0 { return "no container at all" } - return "the container " + strings.Join(names, ", ") + return "the containers " + strings.Join(names, ", ") } // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to diff --git a/internal/bootstrap/phase3_test.go b/internal/bootstrap/phase3_test.go index e5de53d..2b88575 100644 --- a/internal/bootstrap/phase3_test.go +++ b/internal/bootstrap/phase3_test.go @@ -2,10 +2,16 @@ package bootstrap import ( "context" + "encoding/json" + "fmt" + "net/http" "os" "path/filepath" "strings" "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" ) // aCatalogue is a catalogue checkout holding these modules' manifests. @@ -24,32 +30,73 @@ func aCatalogue(t *testing.T, manifests map[string]string) string { return dir } -// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name. -func aBusModule(server string) string { +// aBusModule is the nats module's manifest as far as adoption reads it: its server container, the +// directories it mounts, its certificate through an access, and its user list — the shape of the +// catalogue's modules/nats/module.json. +func aBusModule(l busLayout) string { return `{"module": "nats", "version": "1", "claims": [{"name": "mesh-broker", "scope": "mesh"}], - "bus-users": "/var/lib/nats-module/conf/accounts.conf", - "resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}], + "bus-users": "` + l.Users + `", + "resources": [ + {"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"], + "volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}], + "accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}], "build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}` } -func busStep(t *testing.T, catalogue string) (*asked, error) { +// theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats). +var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats", + Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls", + Users: "/var/lib/nats-module/conf/accounts.conf"} + +// theModulesImage is what the bus runs once the applier recreated it on the module's built image. +const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333" + +// aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the +// module's image, and its monitor says ok. +func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) { + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n", + })} + return runtime, func(_ context.Context, url string) (int, string, error) { + if url != busMonitor { + return 0, "", fmt.Errorf("asked %s", url) + } + return http.StatusOK, `{"status":"ok"}`, nil + } +} + +func busStepWith(t *testing.T, o Options, runtime *asked, + fetch func(context.Context, string) (int, string, error)) ([]string, error) { t.Helper() rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } - runtime := &asked{answer: aMeshThatAgrees(nil)} - control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0} - err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {}) + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} + var said []string + err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch, + func(line string) { said = append(said, line) }) + return said, err +} + +func busStep(t *testing.T, catalogue string) (*asked, error) { + t.Helper() + runtime, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + _, err := busStepWith(t, o, runtime, fetch) return runtime, err } // The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module // the mesh goes on managing, registered from the catalogue — never lavinmq, which left it. func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) { - catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")}) - runtime, _ := busStep(t, catalogue) + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + runtime, err := busStep(t, catalogue) + if err != nil { + t.Fatal(err) + } if !runtime.ran("module add /nats-module.json") { t.Fatalf("the nats module was not registered: %v", runtime.commands) } @@ -74,15 +121,23 @@ func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) { } } -// A bus module whose server is not the foundation's container by name would raise a second bus -// beside the first; it is refused naming both, before anything is registered. -func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) { - catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")}) +// A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus +// beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every +// difference with both sides, before anything is registered. The layout is the one the bundle had +// before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590). +func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) { + before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf", + Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"} + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)}) runtime, err := busStep(t, catalogue) if err == nil { - t.Fatal("a bus module with another server was adopted") + t.Fatal("a bus module kept elsewhere was adopted") } - for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} { + for _, named := range []string{ + `"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`, + `"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`, + `"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`, + } { if !strings.Contains(err.Error(), named) { t.Errorf("the refusal does not name %s: %v", named, err) } @@ -91,3 +146,274 @@ func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) { t.Errorf("something was registered: %v", runtime.commands) } } + +// One field moved is refused as surely as all of them: a module whose streams are elsewhere would be +// raised empty on the bus's own name. +func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) { + moved := theModulesLayout + moved.Streams = "/srv/nats" + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)}) + runtime, err := busStep(t, catalogue) + if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") { + t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err) + } + if strings.Contains(err.Error(), "certificate") { + t.Errorf("the refusal names what agrees: %v", err) + } + if runtime.ran("module add") { + t.Errorf("something was registered: %v", runtime.commands) + } +} + +// The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step +// gives it the setting that maps that to the port the bundle published, 5671 unless given — before the +// push, so the first declaration already names it. +func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) { + for _, c := range []struct { + given int + want string + }{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + runtime, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Ports = FoundationPorts{Bus: c.given} + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + set, pushed := -1, -1 + for i, command := range runtime.commands { + if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 { + set = i + } + if strings.Contains(command, "push anchor") && pushed < 0 { + pushed = i + } + } + if set < 0 || pushed < 0 || set > pushed { + t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "settings nats on anchor: "+c.want) { + t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n")) + } + } +} + +// The one restart is said before it happens, and the step goes on only once the bus runs the module's +// image and answers (novox/hq ADR 0333 §3). +func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image + + // The push is sent; the node takes a few passes to recreate the bus, and the server a few more to + // answer. + pushed, inspected, monitored := false, 0, 0 + runtime := &asked{} + agrees := aMeshThatAgrees(nil) + runtime.answer = func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "push anchor"): + pushed = true + case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"): + if !pushed { + return bundleImage + "\n", nil + } + inspected++ + if inspected < 3 { + return bundleImage + "\n", nil + } + return theModulesImage + "\n", nil + } + return agrees(name, args) + } + fetch := func(context.Context, string) (int, string, error) { + monitored++ + if monitored < 3 { + return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil + } + return http.StatusOK, `{"status":"ok"}`, nil + } + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Wait = 10 * time.Second + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + text := strings.Join(said, "\n") + restart := strings.Index(text, "restarting mesh-broker-nats — once") + answers := strings.Index(text, "bus answers mesh-broker-nats on "+theModulesImage) + adopted := strings.Index(text, "adopted mesh-broker-nats") + if restart < 0 || answers < restart || adopted < answers { + t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text) + } + if inspected < 3 || monitored < 3 { + t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+ + "times, its monitor %d)", inspected, monitored) + } +} + +// A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after +// this one reaches the mesh through that bus, and the push was not applied. +func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n", + })} + fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil } + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Wait = 20 * time.Millisecond + _, err = busStepWith(t, o, runtime, fetch) + if err == nil { + t.Fatal("a bus the node never recreated was taken as adopted") + } + if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") { + t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err) + } +} + +// A re-run after the adoption finds the bus on the module's image already, and says so rather than +// announcing a restart that does not happen. +func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + runtime, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + text := strings.Join(said, "\n") + if strings.Contains(text, "restarting") || !strings.Contains(text, "already restarted mesh-broker-nats") { + t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text) + } +} + +// theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333, +// "How it is checked"): in a merge check the clone the build seat puts beside this repository +// (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue +// beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check +// never skips it. +func theCataloguesBusModule(t *testing.T) []byte { + t.Helper() + const inside = "mesh-catalog/modules/nats/module.json" + if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" { + raw, err := os.ReadFile(filepath.Join(root, inside)) + if err != nil { + t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+ + "cannot be held to the module's: %v", root, err) + } + t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside)) + return raw + } + beside := filepath.Join("..", "..", "..", inside) + raw, err := os.ReadFile(beside) + if err != nil { + t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+ + "bus is not held to the nats module's manifest here; the merge check holds it", beside) + } + t.Logf("judged against the catalogue checked out at %s", beside) + return raw +} + +// **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's +// manifest: the bus's container, its streams, its configuration, its certificate and its user list are +// where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised +// `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590). +func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) { + manifest := theCataloguesBusModule(t) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil { + t.Fatal(err) + } + // And the port the installer's setting maps is the one the module declares (ADR 0333 §2). + var m struct { + Resources []struct { + Type string `json:"type"` + Ports []string `json:"ports"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + declared := false + for _, r := range m.Resources { + for _, p := range r.Ports { + if p == fmt.Sprintf("%d:4222", busModulePort) { + declared = true + } + } + } + if !declared { + t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+ + "to the node's bus port", busModulePort) + } +} + +// The bundle as the installer reads it agrees with the module's layout as this repository states it, +// and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the +// bus reads, and makes the container that reads it re-read it. +func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + ours, err := foundationBusLayout(rewritten.Declaration) + if err != nil { + t.Fatal(err) + } + if ours != theModulesLayout { + t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout) + } + if ours.Users != busAccounts || ours.Container != busContainer { + t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s", + busAccounts, busContainer, ours.Users, ours.Container) + } + if filepath.Dir(ours.Users) != ours.Conf { + t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf) + } +} + +// The bundle's bus configuration and user list are seeds (create-once): an installer run again after +// the adoption applies the bundle again, and must not put the bundle's one user back over the users the +// module writes there (ADR 0333 §4). +func TestTheBundlesBusFilesAreSeeds(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + seeds := map[string]bool{} + for _, r := range rewritten.Declaration.Resources { + if f, ok := r.(*declaration.File); ok && f.CreateOnce { + seeds[f.Path] = true + } + } + for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} { + if !seeds[path] { + t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path) + } + } +} diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index fed87dd..41b33d1 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -72,7 +72,7 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { t.Errorf("the store publishes %v", storeC.Ports) } - bus := containerNamed(r.Declaration, "mesh-broker") + bus := containerNamed(r.Declaration, "mesh-broker-nats") if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" { t.Errorf("the bus publishes %v", bus.Ports) } diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go index 14617b7..b743177 100644 --- a/internal/bootstrap/verify_test.go +++ b/internal/bootstrap/verify_test.go @@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) { // run-once and scheduled shapes are excluded on purpose — a step that has exited is not a // fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the // plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all. - want := []string{"mesh-store", "mesh-broker", "temp-mesh-controller"} + want := []string{"mesh-store", "mesh-broker-nats", "temp-mesh-controller"} if len(verified.Running) != len(want) { t.Fatalf("confirmed %v running, want %v", verified.Running, want) } @@ -132,7 +132,7 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) { // than being told only that something is not what it should be. func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { runtime := &asked{answer: func(_ string, args []string) (string, error) { - if args[0] == "container" && args[len(args)-1] == "mesh-broker" { + if args[0] == "container" && args[len(args)-1] == "mesh-broker-nats" { return "false exited\n", nil } if args[0] == "container" { @@ -146,7 +146,7 @@ func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { if err == nil { t.Fatal("a container that had exited was reported as part of a running foundation") } - if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") { + if !strings.Contains(err.Error(), "mesh-broker-nats") || !strings.Contains(err.Error(), "exited") { t.Errorf("the failure does not say which container is in what state: %v", err) } }