diff --git a/examples/README.md b/examples/README.md index c3a0258..5ad1d1c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,51 +2,37 @@ ## `foundation-first-node.lock` -What a machine must be before a mesh exists — the bootstrap in -[novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole: +The one genesis bundle (novox/hq ADR 0331): what a machine must be before a mesh exists — the +bootstrap in [novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole, on the bus the +mesh runs on: ``` -0 a container runtime +0 a container runtime, and a filter before anything listens 1 the store runs -2 a database per context `inventory` and `identity` +2 a database per context `inventory`, `identity` and `licences` 3 those contexts' schemas mesh-controller migrate -4 the broker runs with a certificate it generated itself -5 the control plane runs mesh-controller serve +4 the bus's certificate mesh-controller broker certificate +5 the bus runs NATS, with the controller's account as its first user +6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS) ``` **A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything else joins the ordinary way. -This file said it stopped at step 4 for longer than that was true, which is its own small lesson: -a comment about what something does not do is a comment nobody updates. +It is a **template**: `mesh-bootstrap` reads it, makes the store's superuser and the controller's +account on the bus, and writes them where the template says `bootstrap` (internal/bootstrap, +rootsecrets.go). A template whose bus connection it does not recognise is refused at preflight, +naming the placeholder it looked for. The bundle that raised the previous broker was deleted with +ADR 0331; there is one bundle, and every lab bed names it. -Build a host carrying it: +Build an installer carrying a builder, and point it at this: ``` -make host SYSTEM=arch BUNDLE=examples/foundation-first-node.lock +make bootstrap IMAGE= +mesh-bootstrap --bundle examples/foundation-first-node.lock … ``` -**The registry address and digests have to be replaced before this is useful.** They are written -as `192.0.2.250:5000/…@sha256:…` because a digest belongs to whatever registry serves it — here, -one a lab scenario raises, which reports its digests when it comes up. That is not a placeholder -to be tidied away: a bundle is built *for a target*, and which registry that target pulls from is -part of the target. - -### What was verified, and how - -On a lab machine confirmed sealed — `curl https://example.com` times out, the lab registry answers -200 — the whole bundle applied from bare: eight resources, `inventory` created and `mesh` nowhere, -the `node` table present with its indexes, the migration recorded, and LavinMQ answering -`lavinmqctl status` with AMQP listening on 5672. - -Three consecutive reconciles after that: **already matches — 8 resource(s) checked**, each time. - -Then the machine was **rebooted**, and everything came back: docker from `boot: enabled`, both -containers because the host creates every container `--restart unless-stopped` -(`internal/apply/apply.go`), the schema intact in its named volume, and reconcile still finding -nothing to do. - -The reboot is worth doing rather than assuming. Nothing in the declaration asks for a container to -return, so that it does is a property of the host, and the only way to know it holds is to take -the machine away and give it back. +**The registry address is the lab's.** The images are written as `192.0.2.250:5000/…@sha256:…`; the +lab rewrites the third-party ones to upstream references by the same digest, and the installer +replaces the control plane's with the image it built. diff --git a/examples/bundle_test.go b/examples/bundle_test.go index 07e75da..85fec2d 100644 --- a/examples/bundle_test.go +++ b/examples/bundle_test.go @@ -7,6 +7,7 @@ package examples import ( "os" + "path/filepath" "sort" "strings" "testing" @@ -48,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin } sort.Strings(images) - want := []string{"lavinmq", "mesh-controller", "postgres"} + want := []string{"mesh-controller", "nats", "postgres"} if strings.Join(images, ",") != strings.Join(want, ",") { t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+ "Adding one is a change to what every first node becomes, and to ADR 0006's "+ @@ -67,3 +68,32 @@ func TestTheControlPlaneIsCarriedToo(t *testing.T) { t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " + "foundation and stop") } + +// One genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331 §2). Two bundles let every bed +// and README name the one that could not raise a mesh, for weeks (issue 146). +func TestThereIsOneGenesisBundle(t *testing.T) { + bundles, err := filepath.Glob("*.lock") + if err != nil { + t.Fatal(err) + } + if len(bundles) != 1 || bundles[0] != "foundation-first-node.lock" { + t.Fatalf("examples/ holds %v; it holds one genesis bundle, foundation-first-node.lock", bundles) + } +} + +// The control plane the bundle raises is given the bus, with the account the installer fills and +// the certificate it pins (ADR 0331 §4); without it, it crash-loops for ever (issue 146). +func TestTheControlPlaneIsGivenTheBus(t *testing.T) { + for _, r := range bundle(t).Resources { + c, ok := r.(*declaration.Container) + if !ok || !strings.Contains(c.Image, "mesh-controller") { + continue + } + if !strings.HasPrefix(c.Env["MESH_BUS_NATS"], "nats://controller:bootstrap@") || c.Env["MESH_BROKER_CERTIFICATE"] == "" { + t.Fatalf("the control plane is given the bus %q and the certificate %q", + c.Env["MESH_BUS_NATS"], c.Env["MESH_BROKER_CERTIFICATE"]) + } + return + } + t.Fatal("nothing in the bundle starts the control plane") +} diff --git a/examples/foundation-first-node-nats.lock b/examples/foundation-first-node-nats.lock deleted file mode 100644 index 25d8462..0000000 --- a/examples/foundation-first-node-nats.lock +++ /dev/null @@ -1,200 +0,0 @@ -// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being -// built (novox/hq ADR 0106, design 25). -// -// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh -// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the -// first one — the controller's own account, at a well-known bootstrap password, exactly as the store -// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and -// from the controller's first composition onward the file is the controller's to write. -// -// The accounts file is its own file beside the server's configuration, because the server's own -// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in -// one directory, of necessity: an include path is resolved relative to the including file's own -// directory, so a server given an absolute one looks for it underneath that directory and refuses to -// start. -// -// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client* -// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and -// authenticates with a password (ADR 0004, design 25 §4). - -{ - "declaration": 1, - "resources": [ - { - "id": "container-runtime", - "type": "package", - "package": "docker" - }, - { - "id": "container-runtime-running", - "type": "service", - "unit": "docker.service", - "state": "running", - "boot": "enabled" - }, - // **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the - // broker are adopted as modules later and so bind to every interface from the moment they - // start; the packet filter that governs who may reach them is a module too, installed a - // dozen steps later. Between the two, a control-node facing the network had its store and - // its bus open to anyone who could reach the machine. So the foundation carries a filter of - // its own — the same table the filter module will replace wholesale once it can derive one: - // drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node - // enrols over, the registry a node pulls from), and let the container runtime's own - // networks through the forward chain so containers keep working. A published container port - // is forwarded, never input (issue 047), which is why the forward chain is where the store's - // and broker's ports are refused from outside — and a container on this machine dialling a - // port this machine publishes reaches it through the runtime's proxy, which IS input, which - // is why the bus and the registry are opened in both chains, exactly as the derived ruleset - // does. - { - "id": "base-filter-package", - "type": "package", - "package": "nftables" - }, - { - "id": "base-filter", - "type": "file", - "path": "/etc/nftables.conf", - "mode": "0644", - "content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n" - }, - { - "id": "base-filter-loaded", - "type": "service", - "unit": "nftables.service", - "state": "running", - "boot": "enabled", - "restart-on": ["base-filter"] - }, - { - "id": "store", - "type": "container", - "name": "mesh-store", - "image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a", - "env": { - "POSTGRES_PASSWORD": "bootstrap", - "PGDATA": "/var/lib/postgresql/data/pgdata" - }, - "ports": ["5432:5432"], - "volumes": ["mesh-store-data:/var/lib/postgresql/data"] - }, - // Over TCP, not the socket. While the store initialises it runs a temporary server on the - // socket ONLY, then stops it and starts the real one — so a socket check passes, the action - // exits happy, and the verify a moment later lands in the gap and fails. The action and its - // verify must ask the same question, or the action can succeed into a state verify rejects. - { - "id": "store-ready", - "type": "action", - "in": "mesh-store", - "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"], -"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"] - }, - { - "id": "inventory-database", - "type": "action", - "in": "mesh-store", - "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"], - "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"] - }, - { - "id": "identity-database", - "type": "action", - "in": "mesh-store", - "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"], - "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"] - }, - // Each context owns its own database (novox/hq ADR 0008). A third one is a third database, - // created the same way and named the same way — which is the whole of adding a context to the - // bootstrap, and is why the count is not something the foundation has an opinion about. - { - "id": "licences-database", - "type": "action", - "in": "mesh-store", - "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"], - "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"] - }, - { - "id": "context-schemas", - "type": "action", - "command": ["docker", "run", "--rm", "--network", "container:mesh-store", - "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", - "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", - "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", - "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", - "migrate"], - "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] - }, - { - "id": "bus-certificate", - "type": "action", - // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the - // broker's image while the broker was one that carried it; the bus that replaced it has a - // shell and no openssl, and no other image the bundle names has one either. So the program - // that needs the certificate writes it — already on this machine, since the schema step ran - // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins - // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh - // to ask an authority of. - // `--user 0:0` because the volume is root's and this image runs as nobody, which is right - // for the long-running control plane and wrong for a one-shot writing into a fresh volume. - "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", - "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", - "broker", "certificate", "--into", "/tls"], - "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", - "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", - "broker", "certificate", "--check", "--into", "/tls"] - }, - { - "id": "bus-conf-dir", - "type": "directory", - "path": "/var/lib/mesh-bus-conf", - "mode": "0700" - }, - { - "id": "bus-conf", - "type": "file", - "path": "/var/lib/mesh-bus-conf/nats.conf", - "mode": "0644", - "content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" - }, - { - "id": "bus-accounts", - "type": "file", - "path": "/var/lib/mesh-bus-conf/accounts.conf", - "mode": "0600", - "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" - }, - { - "id": "broker", - "type": "container", - "name": "mesh-broker", - "image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927", - "ports": ["5671:4222", "127.0.0.1:8222:8222"], - "volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"], - "args": ["-c", "/etc/nats/nats.conf", "-js"] - }, - { - "id": "broker-ready", - "type": "action", - "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], - "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"] - }, - { - "id": "control-plane", - "type": "container", - "name": "mesh-controller", - "image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", - "network": "host", - "args": ["serve"], - "volumes": ["mesh-broker-tls:/broker-tls:ro"], - "env": { - "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", - "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", - "MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", - "MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671", - "MESH_BROKER_ADDRESS": "192.0.2.10:5671", - "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" - } - } - - ] -} diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 860ac91..289e3e2 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -1,36 +1,39 @@ -// foundation-first-node.lock — what a machine must be before a mesh exists. +// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle +// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25). // // The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a -// store, a database per context, those contexts' schemas, the broker, and the control plane -// running on top of them. +// filter, a store, a database per context, those contexts' schemas, the bus with its certificate +// and its first user list, and the control plane running on top of them. // -// It stopped before the control plane once, and this comment said so for longer than it was true. +// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this +// carries the first one — the controller's own account — and the control plane's address names the +// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's +// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap +// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first +// node has enrolled the installer places the list the controller composes; from the bus's adoption +// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5). // -// The broker generates its OWN certificate, in its own image, into a volume it then mounts read -// only. Self-signed, because at this moment there is no mesh to issue one and no public name to -// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint -// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration. +// The accounts file is its own file beside the server's configuration, because the server's own +// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in +// one directory, of necessity: an include path is resolved relative to the including file's own +// directory, so a server given an absolute one looks for it underneath that directory and refuses to +// start. +// +// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client* +// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and +// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through +// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback. // // PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different -// image, and this file is applied on a machine with no mesh to ask about anything. These digests -// belong to the registry the lab raises, which is what a real node pulls from anyway — what is -// required is a reference that is exact and cannot move (novox/hq ADR 0006). -// -// The store waits up to three minutes rather than one. A machine that has just pulled the -// image and is running initdb for the first time can take longer than sixty seconds, and it -// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed, -// which is the worst kind because it teaches people to run things twice. -// -// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited -// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes -// a retry the only available response is a timeout that reports nothing. So the wait now says -// what it saw before giving up. +// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The +// registry prefix is the lab's; the digests are upstream's. // // The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host // creates is owned by root, and the database runs as somebody else inside the container — so it // could not write, and the container crash-looped. A named volume lets the image set up its own // ownership, and outlives the container, which is what you want for the thing holding the mesh's // state. + { "declaration": 1, "resources": [ @@ -139,30 +142,58 @@ "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] }, { - "id": "broker-certificate", + "id": "bus-certificate", "type": "action", - "command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls", - "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336", - "-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"], - "verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls", - "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336", - "-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"] + // **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the + // broker's image while the broker was one that carried it; the bus that replaced it has a + // shell and no openssl, and no other image the bundle names has one either. So the program + // that needs the certificate writes it — already on this machine, since the schema step ran + // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins + // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh + // to ask an authority of. + // `--user 0:0` because the volume is root's and this image runs as nobody, which is right + // for the long-running control plane and wrong for a one-shot writing into a fresh volume. + "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", + "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", + "broker", "certificate", "--into", "/tls"], + "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", + "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", + "broker", "certificate", "--check", "--into", "/tls"] + }, + { + "id": "bus-conf-dir", + "type": "directory", + "path": "/var/lib/mesh-bus-conf", + "mode": "0700" + }, + { + "id": "bus-conf", + "type": "file", + "path": "/var/lib/mesh-bus-conf/nats.conf", + "mode": "0644", + "content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" + }, + { + "id": "bus-accounts", + "type": "file", + "path": "/var/lib/mesh-bus-conf/accounts.conf", + "mode": "0600", + "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, { "id": "broker", "type": "container", "name": "mesh-broker", - "image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336", - "ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"], - "volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"], - "args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"] + "image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927", + "ports": ["5671:4222", "127.0.0.1:8222:8222"], + "volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"], + "args": ["-c", "/etc/nats/nats.conf", "-js"] }, { "id": "broker-ready", "type": "action", - "in": "mesh-broker", - "command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], - "verify": ["lavinmqctl", "status"] + "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], + "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"] }, { "id": "control-plane", @@ -176,8 +207,7 @@ "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", "MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", - "MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/", - "MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672", + "MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671", "MESH_BROKER_ADDRESS": "192.0.2.10:5671", "MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt" } diff --git a/internal/bootstrap/adopted.go b/internal/bootstrap/adopted.go index ed7207c..37ed459 100644 --- a/internal/bootstrap/adopted.go +++ b/internal/bootstrap/adopted.go @@ -185,7 +185,7 @@ func (r Rewritten) declares(id string) bool { // container runtime's configuration whole — and neither is anything the operator installs later. var genesisTakes = map[string]bool{ RegistryModule: true, ControlPlaneModule: true, BuilderModule: true, - "postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true, + "postgres": true, BusModule: true, "mesh-vault": true, "mesh-catalog": true, } // takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 4c4e46a..21a1dee 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -503,7 +503,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro for _, c := range []struct { what, path string made bool - }{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} { + }{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"bus controller", BusControllerFile, creds.BusMade}} { switch { case c.made && o.DryRun: say(fmt.Sprintf(" %-17s would be made and kept at %s (0600)", c.what, c.path)) @@ -513,9 +513,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro say(fmt.Sprintf(" %-17s already at %s — kept", c.what, c.path)) } } - say(fmt.Sprintf(" credentials the template's bootstrap and guest are gone: %d store and %d broker "+ - "connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers", - root.StoreURLs, root.BrokerURLs)) + say(fmt.Sprintf(" credentials the template's bootstrap is gone: %d store and %d bus "+ + "connection(s) rewritten, the store reads its password from a file, the bus is raised with the controller's account", + root.StoreURLs, root.BusURLs)) if rewritten.Renamed { say(fmt.Sprintf(" control plane %s, renamed from %s", rewritten.TempName, rewritten.WasCalled)) @@ -733,7 +733,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro } // ---- broker --------------------------------------------------------------------------- - say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two") + say("broker — the foundation's bus, adopted as the nats module: one bus, not two") if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { return result, failed(StepBroker, err) } diff --git a/internal/bootstrap/control_test.go b/internal/bootstrap/control_test.go index 7df8e9c..2b6fd58 100644 --- a/internal/bootstrap/control_test.go +++ b/internal/bootstrap/control_test.go @@ -30,14 +30,13 @@ const theControlPlaneModule = `{ "inventory": "/var/lib/mesh/mesh-controller/inventory", "identity": "/var/lib/mesh/mesh-controller/identity", "licences": "/var/lib/mesh/mesh-controller/licences", - "broker": "/var/lib/mesh/mesh-controller/broker", - "broker-management": "/var/lib/mesh/mesh-controller/broker-management" + "bus": "/var/lib/mesh/mesh-controller/bus" }, "resources": [ {"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"}, {"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-controller/broker.env", "mode": "0600", - "content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"}, + "content": "MESH_BUS_NATS=${secret:bus}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"}, {"id": "server", "type": "container", "name": "mesh-controller", "image": "` + builtImage + `", "network": "host", "args": ["serve"], @@ -133,11 +132,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { // them at /run/secrets. Matching on the path alone would find nothing and refuse a correct // manifest, which is exactly the ordinary case in the catalogue. for variable, secret := range map[string]string{ - "MESH_STORE_INVENTORY": "inventory", - "MESH_STORE_IDENTITY": "identity", - "MESH_STORE_LICENCES": "licences", - "MESH_BROKER_AMQP": "broker", - "MESH_BROKER_MANAGEMENT": "broker-management", + "MESH_STORE_INVENTORY": "inventory", + "MESH_STORE_IDENTITY": "identity", + "MESH_STORE_LICENCES": "licences", + "MESH_BUS_NATS": "bus", } { if wanted[variable] != secret { t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret) @@ -161,9 +159,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { if err != nil { t.Fatal(err) } - // Three stores and both halves of the broker: everything the foundation made and nothing else. - if len(delivered) != 5 { - t.Fatalf("%d values were delivered, and the foundation names five: %v", + // Three stores and the controller's account on the bus: everything the foundation made and + // nothing else. + if len(delivered) != 4 { + t.Fatalf("%d values were delivered, and the foundation names four: %v", len(delivered), delivered) } for _, secret := range delivered { @@ -177,12 +176,15 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) { // secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh. func TestASecretTheFoundationNeverMadeIsLeftToTheMesh(t *testing.T) { extra := strings.Replace(theControlPlaneModule, - `"broker": "/var/lib/mesh/mesh-controller/broker",`, - `"broker": "/var/lib/mesh/mesh-controller/broker", - "something-new": "/var/lib/mesh/mesh-controller/something-new",`, 1) + `"bus": "/var/lib/mesh/mesh-controller/bus"`, + `"bus": "/var/lib/mesh/mesh-controller/bus", + "something-new": "/var/lib/mesh/mesh-controller/something-new"`, 1) extra = strings.Replace(extra, - `"content": "MESH_BROKER_AMQP=${secret:broker}\n`, - `"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1) + `"content": "MESH_BUS_NATS=${secret:bus}\n`, + `"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BUS_NATS=${secret:bus}\n`, 1) + if extra == theControlPlaneModule { + t.Fatal("the fixture no longer says what this test adds to") + } rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index 7900365..201f49e 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -159,7 +159,8 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla // busAccounts and busContainer are where the installer's bundle raises the bus: the file its // configuration includes, and the container that reads it. The installer raised them, so it is the -// one that knows them (examples/foundation-first-node-nats.lock). +// one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the +// nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5). const ( busAccounts = "/var/lib/mesh-bus-conf/accounts.conf" busContainer = "mesh-broker" diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go index 1d66a25..4d0878f 100644 --- a/internal/bootstrap/enrol_test.go +++ b/internal/bootstrap/enrol_test.go @@ -82,6 +82,11 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) { t.Errorf("a token was issued for a machine that already holds an identity: %v", runtime.commands) } + // Nor are the bus's users placed again: from the adoption on they are the nats module's, and + // the installer's bridge is not run a second time (novox/hq ADR 0331 §5). + if runtime.ran("broker accounts") || runtime.ran("accounts.conf") { + t.Errorf("the bus's users were placed on a re-run: %v", runtime.commands) + } if out.Agent != "already running" { t.Errorf("the host agent is reported as %q", out.Agent) } diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index 2d60811..c8aadbc 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -14,8 +14,8 @@ import ( // Genesis raises a store and a broker before any module system exists, because the control plane // cannot ask provisioning for the store it keeps its own records in or the broker it is reached over // (novox/hq ADR 0006). That leaves two servers behind: the foundation's, and a second one the -// `postgres`/`lavinmq` modules used to raise for other modules to use. This turns the foundation's -// own servers into those modules, so a mesh runs ONE postgres and ONE lavinmq — the control plane's +// `postgres` and bus modules used to raise for other modules to use. This turns the foundation's +// own servers into those modules, so a mesh runs ONE postgres and ONE bus — the control plane's // contexts and the database of each module that asks for one, in the same server (WBS 3.1/3.2). // // **Adopted in place, not replaced.** The control plane is stateless and is swapped for a fresh @@ -155,37 +155,83 @@ func deliverCredential(ctx context.Context, o Options, control controlPlane, mod return nil } -// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same -// shape as InstallStore, for the same reasons. The administrator's password is the one genesis -// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can -// reach the management API as it. +// BusModule is the module the foundation's bus is adopted as (novox/hq ADR 0331 §1). The previous +// broker's module, lavinmq, left the catalogue with the move to one bus (ADR 0131). +const BusModule = "nats" + +// InstallBroker makes the foundation's bus the `nats` module, adopted in place — the same shape as +// InstallStore, for the same reasons: the bus holds the mesh's streams, so it is kept, and from the +// module's first push the bus's user list reaches it in the module's declaration like any module's +// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the +// bridge to here and is not run again. +// +// **What is compared is the container's name, not its image.** The store's module pins the upstream +// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's +// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is +// not the foundation's container by name would raise a second bus beside the first, and is refused +// by name before anything is registered. func InstallBroker(ctx context.Context, o Options, control controlPlane, foundation *declaration.Declaration, say func(string)) error { - const module = "lavinmq" + const module = BusModule manifest, err := readManifest(o.Catalogue, module) if err != nil { return err } - broker, err := brokerIn(foundation) + bus, err := brokerIn(foundation) if err != nil { return err } - if err := serverMatchesFoundation(manifest, broker, module); err != nil { + if err := busServerIsTheFoundations(manifest, bus, module); err != nil { return err } - say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged") - if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)", - func() error { - return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say) - }, - say); err != nil { + say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept") + if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)", + nil, say); err != nil { return err } - say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module") + say(" adopted " + bus.Name + " — the foundation's bus is now the " + module + + " module, and its user list is the module's from here on") return nil } +// busServerIsTheFoundations checks that the bus module's server container is the one the +// foundation raised, by name, so the applier takes it over rather than raising a second bus. +func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error { + var m struct { + Resources []struct { + Type string `json:"type"` + Name string `json:"name"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return fmt.Errorf("the %s module's manifest is not readable: %w", module, err) + } + var names []string + for _, r := range m.Resources { + if r.Type != "container" { + continue + } + if r.Name == bus.Name { + return nil + } + names = append(names, r.Name) + } + return fmt.Errorf( + "the %s module's server is %s, and the foundation's bus is the container %q.\n"+ + "The module adopts the foundation's bus in place, so its server has to carry the name the "+ + "foundation raised; under another name it would raise a second bus beside the first, and "+ + "the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered", + module, describeNames(names), bus.Name) +} + +func describeNames(names []string) string { + if len(names) == 0 { + return "no container at all" + } + return "the container " + strings.Join(names, ", ") +} + // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to // adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider, // and from its first push it keeps the export of every operator-sealed secret on its own disk. @@ -207,9 +253,9 @@ func storeIn(d *declaration.Declaration) (*declaration.Container, error) { return foundationContainer(d, StoreID, "store") } -// brokerIn finds the broker container in the bundle this installer produced. +// brokerIn finds the bus's container in the bundle this installer produced. func brokerIn(d *declaration.Declaration) (*declaration.Container, error) { - return foundationContainer(d, BrokerID, "broker") + return foundationContainer(d, BrokerID, "bus") } func foundationContainer(d *declaration.Declaration, id, what string) (*declaration.Container, error) { diff --git a/internal/bootstrap/phase3_test.go b/internal/bootstrap/phase3_test.go new file mode 100644 index 0000000..e5de53d --- /dev/null +++ b/internal/bootstrap/phase3_test.go @@ -0,0 +1,93 @@ +package bootstrap + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// aCatalogue is a catalogue checkout holding these modules' manifests. +func aCatalogue(t *testing.T, manifests map[string]string) string { + t.Helper() + dir := t.TempDir() + for module, manifest := range manifests { + at := filepath.Join(dir, catalogueDir, module) + if err := os.MkdirAll(at, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name. +func aBusModule(server string) string { + return `{"module": "nats", "version": "1", + "claims": [{"name": "mesh-broker", "scope": "mesh"}], + "bus-users": "/var/lib/nats-module/conf/accounts.conf", + "resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}], + "build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}` +} + +func busStep(t *testing.T, catalogue string) (*asked, error) { + t.Helper() + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0} + err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {}) + return runtime, err +} + +// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module +// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it. +func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")}) + runtime, _ := busStep(t, catalogue) + if !runtime.ran("module add /nats-module.json") { + t.Fatalf("the nats module was not registered: %v", runtime.commands) + } + if runtime.ran("lavinmq") { + t.Errorf("the broker step still speaks of lavinmq: %v", runtime.commands) + } +} + +// A catalogue that holds no nats module fails the step by name, before anything is registered — +// which is what today's step did for lavinmq, silently adopting a module that no longer exists. +func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{"lavinmq": `{"module": "lavinmq", "version": "1"}`}) + runtime, err := busStep(t, catalogue) + if err == nil { + t.Fatal("a catalogue without the nats module was adopted from") + } + if !strings.Contains(err.Error(), "nats module") { + t.Errorf("the refusal does not name the module it looked for: %v", err) + } + if runtime.ran("module add") { + t.Errorf("something was registered: %v", runtime.commands) + } +} + +// A bus module whose server is not the foundation's container by name would raise a second bus +// beside the first; it is refused naming both, before anything is registered. +func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")}) + runtime, err := busStep(t, catalogue) + if err == nil { + t.Fatal("a bus module with another server was adopted") + } + for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} { + if !strings.Contains(err.Error(), named) { + t.Errorf("the refusal does not name %s: %v", named, err) + } + } + if runtime.ran("module add") { + t.Errorf("something was registered: %v", runtime.commands) + } +} diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go index 110b791..8abf637 100644 --- a/internal/bootstrap/ports.go +++ b/internal/bootstrap/ports.go @@ -103,14 +103,18 @@ func (p FoundationPorts) moduleSettings() map[string]map[string]int { out[module][strconv.Itoa(declared)] = given } add("postgres", d.Store, p.Store) - add("lavinmq", d.Bus, p.Bus) - add("lavinmq", d.AMQP, p.AMQP) - add("lavinmq", d.Management, p.Management) + // The bus's module declares the server's own port, and the foundation published the bus on the + // node's: adopted in place, it keeps the port every token already names (ADR 0331 §1). + add(BusModule, busModulePort, p.Bus) add(RegistryModule, d.Registry, p.Registry) add(ForgeModule, d.Packages, p.Packages) return out } +// busModulePort is the machine's port the nats module declares for the bus (mesh-catalog +// modules/nats, "4222:4222"). +const busModulePort = 4222 + // PortsSetting is the controller's settings key for a module's given ports. const PortsSetting = "ports" @@ -200,10 +204,10 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe if p.Store != d.Store { replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port") } - if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management { - replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`, - fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management), - "the broker's published ports") + if p.Bus != d.Bus { + replace(`"ports": ["5671:4222", "127.0.0.1:8222:8222"]`, + fmt.Sprintf(`"ports": ["%d:4222", "127.0.0.1:8222:8222"]`, p.Bus), + "the bus's published ports") } if err != nil { return out, err @@ -221,9 +225,9 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe value := control.Env[key] now := value now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store)) - now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP)) - if strings.HasSuffix(now, "@127.0.0.1:15672") { - now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management) + // The bus, as the control plane dials it (MESH_BUS_NATS) on the machine's loopback. + if strings.HasSuffix(now, "@127.0.0.1:5671") { + now = strings.TrimSuffix(now, "5671") + strconv.Itoa(p.Bus) } if key == brokerAddressVar { if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" { diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index b83e370..50454b4 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -31,7 +31,7 @@ func producedBundle(t *testing.T) Rewritten { if err != nil { t.Fatal(err) } - if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { + if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil { t.Fatal(err) } return r @@ -72,9 +72,9 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { t.Errorf("the store publishes %v", storeC.Ports) } - broker := containerNamed(r.Declaration, "mesh-broker") - if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" { - t.Errorf("the broker publishes %v", broker.Ports) + bus := containerNamed(r.Declaration, "mesh-broker") + if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" { + t.Errorf("the bus publishes %v", bus.Ports) } control, err := controlPlaneIn(r.Declaration) if err != nil { @@ -85,9 +85,8 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { t.Errorf("%s still dials %s", key, value) } } - if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") || - !strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") { - t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"]) + if !strings.HasSuffix(control.Env[BusVar], "@127.0.0.1:5771") { + t.Errorf("the control plane dials the bus at %s", control.Env[BusVar]) } if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" { t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress) diff --git a/internal/bootstrap/preflight.go b/internal/bootstrap/preflight.go index f3502a0..0e4ca46 100644 --- a/internal/bootstrap/preflight.go +++ b/internal/bootstrap/preflight.go @@ -12,6 +12,28 @@ import ( "github.com/novox/mesh-host/internal/profile" ) +// CheckTemplate is what preflight asks of the bundle template before anything is changed: that it +// is a declaration, that it raises a control plane, and that it names the bus the way this +// installer fills it (novox/hq ADR 0331 §3, §4). +// +// Parsed here as well as at the rewrite, because a template that is not a declaration should cost +// a second rather than an image load and a written file. And a control plane given no bus never +// answers: the verify step would wait its whole timeout to say so, and a template like that is a +// fault in the bundle, said at once. +func CheckTemplate(template []byte) (*declaration.Declaration, error) { + parsed, err := declaration.ParseFileTrusted(template) + if err != nil { + return nil, fmt.Errorf("the bundle template is not a declaration: %w", err) + } + if _, err := controlPlaneIn(parsed); err != nil { + return nil, err + } + if err := BusPlaceholders(template, parsed); err != nil { + return nil, err + } + return parsed, nil +} + // DefaultRegistry is where an image reference that names no host comes from. const DefaultRegistry = "registry-1.docker.io:443" @@ -76,13 +98,8 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte "it. Point --bundle at one; mesh-host's examples/foundation-first-node.lock is "+ "the shape", err) } - // Parsed here as well as at the rewrite, because a template that is not a declaration should - // cost a second rather than an image load and a written file. - parsed, err := declaration.ParseFileTrusted(template) + parsed, err := CheckTemplate(template) if err != nil { - return nil, fmt.Errorf("the bundle template is not a declaration: %w", err) - } - if _, err := controlPlaneIn(parsed); err != nil { return nil, err } say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources))) diff --git a/internal/bootstrap/preflight_test.go b/internal/bootstrap/preflight_test.go index 3321ef5..9dcc644 100644 --- a/internal/bootstrap/preflight_test.go +++ b/internal/bootstrap/preflight_test.go @@ -149,3 +149,21 @@ func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) { } } } + +// A template whose temporary control plane is given no bus is refused at preflight, before anything +// is changed, rather than raised and waited on at the verify step until its timeout (novox/hq issue +// 146, ADR 0331 §4): that control plane says it has no MESH_BUS_NATS every few seconds, for ever. +func TestATemplateWhoseControlPlaneHasNoBusIsRefusedBeforeAnythingIsRaised(t *testing.T) { + template := string(theRealBundle(t)) + if _, err := CheckTemplate([]byte(template)); err != nil { + t.Fatalf("the shipped bundle is refused at preflight: %v", err) + } + const bus = `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",` + if !strings.Contains(template, bus) { + t.Fatalf("the shipped bundle does not give its control plane %s", bus) + } + _, err := CheckTemplate([]byte(strings.Replace(template, bus, "", 1))) + if err == nil || !strings.Contains(err.Error(), BusVar) { + t.Fatalf("a control plane with no bus was let through preflight: %v", err) + } +} diff --git a/internal/bootstrap/rewrite.go b/internal/bootstrap/rewrite.go index 456b219..a3ede6a 100644 --- a/internal/bootstrap/rewrite.go +++ b/internal/bootstrap/rewrite.go @@ -99,7 +99,7 @@ type Rewritten struct { // migration pointing at an image no registry serves, and the apply would fail in the middle — // after the store is up, before the broker. They are one image and they move together. // -// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references +// **Third-party images are not touched.** postgres and nats keep the `name@sha256:` references // the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is // checked afterwards rather than merely intended: the produced bundle is re-parsed and every other // container's image is compared against what it was. diff --git a/internal/bootstrap/rewrite_test.go b/internal/bootstrap/rewrite_test.go index 0ad169c..d9963e2 100644 --- a/internal/bootstrap/rewrite_test.go +++ b/internal/bootstrap/rewrite_test.go @@ -183,7 +183,7 @@ func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) { if err != nil { t.Fatal(err) } - const remembered = "NAMED VOLUME" + const remembered = "A filter before anything listens" if !strings.Contains(string(out.Bundle), remembered) { t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered) } diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go index 4b5ec7c..9e63874 100644 --- a/internal/bootstrap/rootsecrets.go +++ b/internal/bootstrap/rootsecrets.go @@ -4,49 +4,58 @@ import ( "bytes" "context" "crypto/rand" - "crypto/sha256" "encoding/base64" - "encoding/hex" "fmt" "os" "path/filepath" "strings" + "golang.org/x/crypto/bcrypt" + "github.com/novox/mesh-host/internal/declaration" ) // The mesh's root credentials, made at genesis rather than copied from the template. // -// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq -// issue 071). The store's superuser and the broker's administrator are the two credentials every -// other one rests on, and they were the two that were not secret: constants in a file anybody can -// read, carried into the mesh by `secret accept` and marked as something the mesh must never -// replace — which is correct for a credential that already created the databases, and made the -// well-known value permanent. +// **The template carries `bootstrap`, and a mesh raised from it kept it** (novox/hq issue 071). The +// store's superuser and the controller's account on the bus are the two credentials every other one +// rests on, and they were the two that were not secret: constants in a file anybody can read, +// carried into the mesh by `secret accept` and marked as something the mesh must never replace — +// which is correct for a credential that already created the databases, and made the well-known +// value permanent. // -// So the installer makes them. Two random values, **made once and kept on this machine** at the -// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three -// adopts the store and the broker, `secret accept` carries in exactly the value the servers were -// raised with, and the host's later write of the sealed secret lands the same bytes in the same -// file. A second run finds the files and changes nothing, which is what lets the installer say -// "already done" about a store it must not restart. +// So the installer makes them. Two random values, **made once and kept on this machine** — the +// store's at the path the postgres module declares as its own secret, so that when phase three +// adopts the store `secret accept` carries in exactly the value the server was raised with; the +// bus's beside the other things only genesis holds. A second run finds the files and changes +// nothing, which is what lets the installer say "already done" about a store it must not restart. // // **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a // container's environment is in `docker inspect` for ever; the module that adopts the store // declares the same file mount, so the two specs are one and the applier reconciles rather than -// recreates (phase3.go). The broker has no such file: its image's default administrator is changed -// in place by an action once the broker answers, and the produced bundle carries that action. +// recreates (phase3.go). +// +// **The bus has no administrator to change; it has a user list** (novox/hq ADR 0331 §3). The +// bundle carries the first list — the controller's own account at `bootstrap` — and the +// controller's address names the same account. Both are the bundle's placeholders: the address's +// `nats://controller:bootstrap@` and the controller's entry in the carried list. The installer +// puts the made password in the first and its bcrypt hash in the second, so the bus is raised with +// the account the controller dials with. The controller records a hash of what it actually uses +// (mesh-controller internal/broker, CredentialIn), so its own compositions carry the same account +// onward, and the permanent controller is given it by `secret accept` (control.go deliverStores). const ( // StoreSuperuserFile is where the store's superuser password lives on the machine — the // postgres module's own-secret path, so genesis and adoption write the same file. StoreSuperuserFile = "/var/lib/postgres/superuser.secret" - // BrokerAdminFile is the same for the broker's administrator — the lavinmq module's. - BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret" - // BrokerAdminUser is the broker's administrator. The image's default account, kept by name - // and given a password that is not the image's default; a renamed account would have to be - // created before anything can authenticate, and the thing that creates accounts is the thing - // that has to authenticate first. - BrokerAdminUser = "guest" + // BusControllerFile is where the controller's password on the bus lives on the machine. Not a + // module's path: no module reads it — the permanent controller is handed the address the + // temporary one was given — and genesis keeps it only so a second run puts the same account + // in the same two places. + BusControllerFile = "/var/lib/mesh-genesis/bus-controller.secret" + // BusControllerHashFile is the bcrypt hash of it, kept with it. bcrypt salts every hash, so a + // hash made afresh on every run would rewrite the bus's user list on every run; kept, the + // produced bundle is the same bundle twice. + BusControllerHashFile = BusControllerFile + ".bcrypt" storeSuperuserMount = "/run/secrets/superuser" @@ -55,15 +64,81 @@ const ( templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]` templateStoreURL = "postgres:bootstrap@" - templateBrokerURL = "guest:guest@" - templateBrokerReady = "\"verify\": [\"lavinmqctl\", \"status\"]\n }," - brokerAdminMarker = "/var/lib/lavinmq/.mesh-admin" + + // BusURLPlaceholder is the controller's account in its bus address, as the template names it. + BusURLPlaceholder = "nats://" + busControllerUser + ":bootstrap@" + // BusUserPlaceholder is the controller's entry in the user list the template carries, up to + // its password's hash — written as it stands inside the bundle's JSON string, quotes escaped. + BusUserPlaceholder = `{ user: \"` + busControllerUser + `\", password: \"` + busControllerUser = "controller" ) -// RootCredentials are the two values, and whether this run made them. +// BusVar is where the control plane finds the mesh's bus, with its own account (mesh-controller +// internal/broker NATSVar), and CertificateVar the certificate it pins the bus to (ADR 0004). +const ( + BusVar = "MESH_BUS_NATS" + CertificateVar = "MESH_BROKER_CERTIFICATE" +) + +// BusPlaceholders refuses a template whose bus connection this installer does not recognise, +// naming the placeholder it looked for (novox/hq ADR 0331 §3, §4). +// +// Asked at preflight, before anything is changed, and again where the bundle is rewritten. A +// template with no `MESH_BUS_NATS` raises a control plane that cannot reach the mesh's bus and +// says so every few seconds for ever (issue 146): the verify step would wait out its whole timeout +// for an answer that cannot come. That is a fault in the bundle, and it is found here instead. +func BusPlaceholders(template []byte, d *declaration.Declaration) error { + control, err := controlPlaneIn(d) + if err != nil { + return err + } + address := strings.TrimSpace(control.Env[BusVar]) + if address == "" { + return fmt.Errorf("the template's control plane (%q) is given no %s, so it could not reach the mesh's "+ + "bus and would never answer. A genesis template names the bus as %s
, which this "+ + "installer fills with the account it makes; mesh-host's examples/foundation-first-node.lock "+ + "is the shape", control.Name, BusVar, BusURLPlaceholder) + } + if !strings.HasPrefix(address, BusURLPlaceholder) { + return fmt.Errorf("the template's control plane reaches the bus at %s, and this installer looked for "+ + "%s
there: the account it makes goes where that placeholder is, and it will not "+ + "guess where else a bus connection could be", redactURL(address), BusURLPlaceholder) + } + if strings.TrimSpace(control.Env[CertificateVar]) == "" { + return fmt.Errorf("the template's control plane is given no %s, so it could not pin the bus's "+ + "certificate (novox/hq ADR 0004) and would refuse to dial it", CertificateVar) + } + if n := bytes.Count(template, []byte(BusUserPlaceholder)); n != 1 { + return fmt.Errorf("the template's user list for the bus names the controller's account %d times, and "+ + "this installer looked for it once, as %s: the bus has to be raised with the account the "+ + "controller dials with, and that is where it goes", n, BusUserPlaceholder) + } + return nil +} + +// redactURL is a bus address with any password in it replaced, for an error that quotes it. +func redactURL(address string) string { + scheme, rest, has := strings.Cut(address, "://") + if !has { + rest, scheme = address, "" + } + at := strings.LastIndex(rest, "@") + user, _, hasPassword := strings.Cut(rest[:max(at, 0)], ":") + if at < 0 || !hasPassword { + return address + } + if scheme != "" { + scheme += "://" + } + return scheme + user + ":…" + rest[at:] +} + +// RootCredentials are the two values, and whether this run made them. BusHash is the bcrypt hash of +// Bus, as the bus's user list carries it. type RootCredentials struct { - Store, Broker string - StoreMade, BrokerMade bool + Store, Bus string + BusHash string + StoreMade, BusMade bool } // RootSecrets reads the credentials this machine already holds, or makes them. @@ -71,17 +146,44 @@ type RootCredentials struct { // A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the // real one, and a machine that was only asked is not left holding half a genesis. func RootSecrets(dryRun bool) (RootCredentials, error) { + return rootSecretsAt(StoreSuperuserFile, BusControllerFile, BusControllerHashFile, dryRun) +} + +func rootSecretsAt(storeFile, busFile, hashFile string, dryRun bool) (RootCredentials, error) { var out RootCredentials var err error - if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil { + if out.Store, out.StoreMade, err = keptOrMade(storeFile, dryRun); err != nil { return out, err } - if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil { + if out.Bus, out.BusMade, err = keptOrMade(busFile, dryRun); err != nil { + return out, err + } + if out.BusHash, err = keptHash(hashFile, out.Bus, dryRun); err != nil { return out, err } return out, nil } +// keptHash is the bcrypt hash of password kept at path, made again only when the kept one is +// missing or no longer the hash of this password. +func keptHash(path, password string, dryRun bool) (string, error) { + if kept, err := readCredentialFile(path); err == nil { + if bcrypt.CompareHashAndPassword([]byte(kept), []byte(password)) == nil { + return kept, nil + } + } else if !os.IsNotExist(err) { + return "", err + } + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", err + } + if dryRun { + return string(hash), nil + } + return string(hash), writeCredentialFile(path, string(hash)) +} + func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { value, err = readCredentialFile(path) if err == nil { @@ -97,8 +199,13 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { if dryRun { return value, true, nil } + return value, true, writeCredentialFile(path, value) +} + +// writeCredentialFile writes a credential as genesis keeps it. +func writeCredentialFile(path, value string) error { if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return "", false, err + return err } // Written whole and renamed into place, at 0600, owned by whoever runs the installer — root, // which is also who the host runs as when it later writes the sealed copy here. @@ -112,12 +219,9 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) { // tolerates either ending, so a file an earlier genesis wrote still reads. tmp := path + ".genesis" if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil { - return "", false, err + return err } - if err := os.Rename(tmp, path); err != nil { - return "", false, err - } - return value, true, nil + return os.Rename(tmp, path) } // readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone. @@ -134,13 +238,6 @@ func readCredentialFile(path string) (string, error) { return value, nil } -// credentialFingerprint names a credential without being one — what the broker-admin action -// leaves on the broker's volume, so its verify holds for this value and not for any value. -func credentialFingerprint(value string) string { - sum := sha256.Sum256([]byte(value)) - return hex.EncodeToString(sum[:8]) -} - // freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40 // characters, URL-safe — it lands inside connection strings. func freshSecret() (string, error) { @@ -153,55 +250,48 @@ func freshSecret() (string, error) { // Masking makes a reporter that never says the credentials this run made. // -// The applier reports each action with its command line, and two of them now carry a real -// password — the context schemas' connection strings and the broker's change_password. Those -// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The -// exact values are known here, so they are replaced wherever they appear, in every line said. +// The applier reports each action with its command line, and the context schemas' connection +// strings carry a real password. Those lines go to a terminal and to whatever keeps the +// transcript, which for the lab is a file. The exact values are known here, so they are replaced +// wherever they appear, in every line said. func Masking(say func(string), c RootCredentials) func(string) { - replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…") - if c.Store == "" || c.Broker == "" { + if c.Store == "" || c.Bus == "" { return say } + replacer := strings.NewReplacer(c.Store, "…", c.Bus, "…") return func(line string) { say(replacer.Replace(line)) } } -// RefuseExistingServers stops a run that would put a made credential in front of a server raised +// RefuseExistingServers stops a run that would put a made credential in front of a store raised // by an earlier installer with the template's. // -// The store's password is set by initdb, once, on an empty volume; the broker's by the action -// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no -// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a -// bundle that dials with passwords the servers do not have — failing three steps later, in the -// schemas, with nothing pointing back here. Refused by name instead, with the way forward. +// The store's password is set by initdb, once, on an empty volume. A machine that already holds +// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new +// value here would make a bundle that dials with a password the server does not have — failing +// three steps later, in the schemas, with nothing pointing back here. Refused by name instead, +// with the way forward. +// +// **The bus is not asked.** Its accounts live in the user list beside its configuration, which +// the bundle writes and the bus re-reads, not in its data volume — so a bus raised earlier takes +// the made account as readily as a new one. func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error { - for _, check := range []struct { - made bool - volume string - what string - file string - }{ - {c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile}, - {c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile}, - } { - if !check.made { - continue - } - if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil { - continue // no such volume: a fresh machine, which is the case this installer makes - } - return fmt.Errorf( - "this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+ - "by an earlier installer with the template's password. A new one made here would not open it. "+ - "Put the password the %s has into %s (0600, the value alone) and run again; then change it "+ - "on the server and accept the new value — this installer does not rotate a running %s", - check.what, check.volume, check.file, check.what, check.file, check.what) + if !c.StoreMade { + return nil } - return nil + if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil { + return nil // no such volume: a fresh machine, which is the case this installer makes + } + return fmt.Errorf( + "this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+ + "by an earlier installer with the template's password. A new one made here would not open it. "+ + "Put the password the store has into %s (0600, the value alone) and run again; then change it "+ + "on the server and accept the new value — this installer does not rotate a running store", + StoreSuperuserFile, StoreSuperuserFile) } // RootRewrite says what RewriteRoot did to the bundle. type RootRewrite struct { - StoreURLs, BrokerURLs int + StoreURLs, BusURLs int } // RewriteRoot puts the made credentials into the produced bundle, in place of the template's. @@ -225,37 +315,30 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { "the store's volumes"); err != nil { return out, err } - // Everything that dials the store or the broker with the template's credentials. + // Everything that dials the store with the template's credentials. out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL)) if out.StoreURLs == 0 { return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL) } bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@")) - out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL)) - if out.BrokerURLs == 0 { - return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL) - } - bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@")) - // The broker's administrator, changed once the broker answers and before anything dials it. - // Verified by a marker on the broker's own data volume holding this password's fingerprint — - // the image carries nothing that can try a password from inside, and a marker that merely - // existed would let a regenerated password go unapplied for ever. What proves the password - // works is the control plane answering over it, a few resources later. The marker ends in a - // newline because the verify reads it with the shell's `read`, which fails at end of file - // without one — an action that ran and a verify that said no, once, in the lab. - fp := credentialFingerprint(c.Broker) - action := templateBrokerReady + "\n" + - " {\n" + - " \"id\": \"broker-admin\",\n" + - " \"type\": \"action\",\n" + - " \"in\": \"mesh-broker\",\n" + - " \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" + - " \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" + - " }," - if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil { + // The bus: the controller's account, in its address and in the user list the bus is raised + // with, the same account in both (novox/hq ADR 0331 §3). + if err := BusPlaceholders(bundle, r.Declaration); err != nil { return out, err } + out.BusURLs = bytes.Count(bundle, []byte(BusURLPlaceholder)) + bundle = bytes.ReplaceAll(bundle, []byte(BusURLPlaceholder), + []byte("nats://"+busControllerUser+":"+c.Bus+"@")) + at := bytes.Index(bundle, []byte(BusUserPlaceholder)) + len(BusUserPlaceholder) + end := bytes.Index(bundle[at:], []byte(`\"`)) + if end < 0 { + return out, fmt.Errorf("the controller's entry in the template's user list for the bus has no end to its password") + } + if !strings.HasPrefix(c.BusHash, "$2") { + return out, fmt.Errorf("the controller's bus password was given no bcrypt hash, so the bus could not be raised with it; this is the installer's fault") + } + bundle = append(append(append([]byte{}, bundle[:at]...), c.BusHash...), bundle[at+end:]...) parsed, err := declaration.ParseFileTrusted(bundle) if err != nil { diff --git a/internal/bootstrap/rootsecrets_test.go b/internal/bootstrap/rootsecrets_test.go index d64379b..c0db52a 100644 --- a/internal/bootstrap/rootsecrets_test.go +++ b/internal/bootstrap/rootsecrets_test.go @@ -5,13 +5,24 @@ import ( "strings" "testing" + "golang.org/x/crypto/bcrypt" + "github.com/novox/mesh-host/internal/declaration" ) -// The produced bundle carries no well-known credential: the store reads its password from the -// file genesis made, every connection string names the made values, and the broker's default -// administrator is changed by an action before anything dials it (novox/hq issue 071). -func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { +// testCredentials are made values with a real bcrypt hash of the bus password, as RootSecrets makes. +func testCredentials(t *testing.T, store, bus string) RootCredentials { + t.Helper() + hash, err := bcrypt.GenerateFromPassword([]byte(bus), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + return RootCredentials{Store: store, Bus: bus, BusHash: string(hash)} +} + +// theExample is the one genesis bundle, rewritten for an image as the installer does. +func theExample(t *testing.T) Rewritten { + t.Helper() template, err := os.ReadFile("../../examples/foundation-first-node.lock") if err != nil { t.Skip("no example bundle beside this checkout") @@ -20,81 +31,142 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { if err != nil { t.Fatal(err) } - creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"} + return r +} + +// The produced bundle carries no well-known credential: the store reads its password from the +// file genesis made, and the controller's account on the bus is the made one in both places the +// bundle names it — the address the control plane dials and the user list the bus is raised with +// (novox/hq issue 071, ADR 0331 §3). +func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { + r := theExample(t) + creds := testCredentials(t, "STORE-PW-40-characters-of-random-base64u", "BUS-PW-40-characters-of-random-base64uuu") got, err := RewriteRoot(&r, creds) if err != nil { t.Fatal(err) } text := string(r.Bundle) - for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} { + for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "controller:bootstrap@"} { if strings.Contains(text, gone) { t.Errorf("the produced bundle still says %s", gone) } } - if got.StoreURLs < 3 || got.BrokerURLs < 2 { - t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs) + if got.StoreURLs < 3 || got.BusURLs != 1 { + t.Errorf("rewrote %d store and %d bus connections; the template has three and one", got.StoreURLs, got.BusURLs) } - var store, action bool + var store, accounts, control bool for _, res := range r.Declaration.Resources { switch x := res.(type) { case *declaration.Container: - if x.Name != "mesh-store" { + switch x.Name { + case "mesh-store": + store = true + if _, has := x.Env["POSTGRES_PASSWORD"]; has { + t.Error("the store still takes its password from its environment") + } + if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount { + t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"]) + } + if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) { + t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes) + } + case r.TempName: + control = true + if want := "nats://controller:" + creds.Bus + "@127.0.0.1:5671"; x.Env[BusVar] != want { + t.Errorf("the temporary control plane dials the bus at %q, not with the made account", x.Env[BusVar]) + } + } + case *declaration.File: + if x.ID != "bus-accounts" { continue } - store = true - if _, has := x.Env["POSTGRES_PASSWORD"]; has { - t.Error("the store still takes its password from its environment") + accounts = true + hash := hashOf(t, x.Content, "controller") + if bcrypt.CompareHashAndPassword([]byte(hash), []byte(creds.Bus)) != nil { + t.Errorf("the bus is raised with a controller account whose hash %q is not the made password's", hash) } - if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount { - t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"]) - } - if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) { - t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes) - } - case *declaration.Action: - if x.ID != "broker-admin" { - continue - } - action = true - if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") { - t.Errorf("the broker-admin action is %v in %q", x.Command, x.In) + if bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")) == nil { + t.Error("the bus is still raised with the controller at bootstrap") } } } - if !store || !action { - t.Fatalf("store=%v action=%v", store, action) - } - // The order matters: the broker's password changes after it answers and before the control - // plane, which dials it with the new one, is raised. - var readyAt, adminAt, controlAt int - for i, res := range r.Declaration.Resources { - switch res.Identity() { - case "broker-ready": - readyAt = i - case "broker-admin": - adminAt = i - case "control-plane": - controlAt = i - } - } - if !(readyAt < adminAt && adminAt < controlAt) { - t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt) + if !store || !accounts || !control { + t.Fatalf("store=%v accounts=%v control=%v", store, accounts, control) } } -// A template that no longer says what this expects is refused, not half-rewritten. -func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) { +// hashOf is the password hash a NATS user list gives one user. +func hashOf(t *testing.T, list, user string) string { + t.Helper() + marker := `{ user: "` + user + `", password: "` + at := strings.Index(list, marker) + if at < 0 { + t.Fatalf("the user list names no %s:\n%s", user, list) + } + rest := list[at+len(marker):] + return rest[:strings.Index(rest, `"`)] +} + +// The bus's placeholders are what the installer looks for, and a template naming no bus connection +// it knows is refused at preflight, naming the placeholder (novox/hq ADR 0331 §3, §4). +func TestABundleNamingNoBusConnectionItKnowsIsRefusedByItsPlaceholder(t *testing.T) { template, err := os.ReadFile("../../examples/foundation-first-node.lock") if err != nil { t.Skip("no example bundle beside this checkout") } - changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1) - r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32)) + parse := func(b string) *declaration.Declaration { + d, err := declaration.ParseFileTrusted([]byte(b)) + if err != nil { + t.Fatal(err) + } + return d + } + if err := BusPlaceholders(template, parse(string(template))); err != nil { + t.Fatalf("the shipped bundle is refused: %v", err) + } + for _, c := range []struct{ what, from, to, names string }{ + {"no bus address", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`, "", BusVar}, + {"the previous broker's", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671"`, + `"MESH_BUS_NATS": "amqp://guest:guest@127.0.0.1:5671"`, BusURLPlaceholder}, + {"no pinned certificate", `"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"`, `"MESH_OTHER": "x"`, CertificateVar}, + {"no controller in the user list", `{ user: \"controller\"`, `{ user: \"someone\"`, BusUserPlaceholder}, + } { + if !strings.Contains(string(template), c.from) { + t.Fatalf("%s: the bundle does not say %s", c.what, c.from) + } + changed := strings.Replace(string(template), c.from, c.to, 1) + err := BusPlaceholders([]byte(changed), parse(changed)) + if err == nil { + t.Errorf("%s: accepted", c.what) + continue + } + if !strings.Contains(err.Error(), c.names) { + t.Errorf("%s: refused without naming %s: %v", c.what, c.names, err) + } + if strings.Contains(err.Error(), "guest:guest") { + t.Errorf("%s: the refusal quotes a password: %v", c.what, err) + } + } +} + +// The account is made once, with its hash, and a second run puts the same bytes in the bundle — a +// hash salted afresh every run would rewrite the bus's user list every run. +func TestTheBusAccountIsKeptWithItsHash(t *testing.T) { + dir := t.TempDir() + first, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false) if err != nil { t.Fatal(err) } - if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil { - t.Fatal("a template with an unknown store password was rewritten") + if !first.BusMade || len(first.Bus) != 40 || bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) != nil { + t.Fatalf("first run: made=%v hash ok=%v", first.BusMade, bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) == nil) + } + second, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false) + if err != nil { + t.Fatal(err) + } + if second.BusMade || second.Bus != first.Bus || second.BusHash != first.BusHash { + t.Fatalf("a second run changed the bus account: made=%v same=%v same hash=%v", + second.BusMade, second.Bus == first.Bus, second.BusHash == first.BusHash) } } @@ -125,60 +197,25 @@ func TestRootSecretsAreKeptAcrossRuns(t *testing.T) { // Nothing the installer says after making the credentials contains them. func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) { var said []string - say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"}) + say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Bus: "BUS-PW"}) say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)") - say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)") + say("failed control-plane (MESH_BUS_NATS=nats://controller:BUS-PW@127.0.0.1:5671)") for _, l := range said { - if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") { + if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BUS-PW") { t.Errorf("said a credential: %s", l) } } - if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") { + if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "controller:…@") { t.Errorf("the lines were not the same lines with the values masked: %v", said) } } -// The broker-admin marker is written with a line ending, because the verify reads it with `read`. -func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) { - template, err := os.ReadFile("../../examples/foundation-first-node.lock") - if err != nil { - t.Skip("no example bundle beside this checkout") - } - r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) - if err != nil { - t.Fatal(err) - } - if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { - t.Fatal(err) - } - for _, res := range r.Declaration.Resources { - a, ok := res.(*declaration.Action) - if !ok || a.ID != "broker-admin" { - continue - } - cmd := strings.Join(a.Command, " ") - if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") { - t.Errorf("the marker is written without a line ending: %s", cmd) - } - if !strings.Contains(strings.Join(a.Verify, " "), "read m <") { - t.Errorf("the verify does not read the marker: %v", a.Verify) - } - } -} - // The foundation filters before anything listens: nftables is in place before the store, and its // rules refuse the store's and broker's client ports from outside while keeping ssh, the bus a node // enrols over and the registry a node pulls from (novox/hq issue 054). func TestTheFoundationFiltersBeforeAnythingListens(t *testing.T) { - template, err := os.ReadFile("../../examples/foundation-first-node.lock") - if err != nil { - t.Skip("no example bundle beside this checkout") - } - r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32)) - if err != nil { - t.Fatal(err) - } - if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil { + r := theExample(t) + if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil { t.Fatal(err) } var filterAt, loadedAt, storeAt, brokerAt = -1, -1, -1, -1