diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 4bc7517..369a744 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh --tools-source the repository the shared base is built from --tools-ref what of it to build (default main) + --tools-go-source the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source) + --tools-go-ref what of it to build (default main) --catalog-source the catalogue REPOSITORY, for building its modules; --catalog is the checkout that says what they are --catalog-ref what of it to build (default main) @@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { "the repository the shared base is built from") set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref, "what of it to build (default main)") + set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository, + "the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)") + set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref, + "what of it to build (default main)") set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository, "the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are") set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref, diff --git a/cmd/mesh-host/local_network_test.go b/cmd/mesh-host/local_network_test.go new file mode 100644 index 0000000..3a7af99 --- /dev/null +++ b/cmd/mesh-host/local_network_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "testing" + + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/outward" + "github.com/novox/mesh-host/internal/profile" +) + +func reportOn(networks ...outward.Network) link.Report { + return link.Report{Outward: []string{"eth0"}, + Profile: profileAsReported(profile.Profile{Machine: &profile.Machine{Networks: networks}})} +} + +// **A change of the networks a machine is attached to is said unasked** (novox/hq ADR 0340 §7): a port +// reached from the local network is admitted from them, so a laptop that joins another network is said at +// the next reconcile, not at the next push — and the same networks again are not news. +func TestAChangedNetworkIsSaidUnasked(t *testing.T) { + w := &adoptionWatch{} + home := reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"}) + w.said(home) + if w.differs(reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})) { + t.Fatal("the same networks read as a change, so the machine would speak on every reconcile") + } + if !w.differs(reportOn(outward.Network{Link: "eth0", Network: "10.20.0.0/16"})) { + t.Fatal("a machine that moved to another network does not say so until it is pushed to") + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 8c79ab3..eb779bd 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1399,6 +1399,12 @@ func adoptionFingerprint(r link.Report) string { // it could only speak when a declaration arrived, and a declaration cannot be composed until it // has spoken: a machine waiting for a push that is waiting for the machine. parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")} + // **And the networks those links are on** (novox/hq ADR 0340 §7): a port reached from the local + // network is admitted from them, so a laptop that joins another network, or a lease that moves, is + // said at the next reconcile rather than at the next push or the next day. + if networks := attachedNetworksOf(r); networks != "" { + parts = append(parts, "networks="+networks) + } // And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk // waiting on this machine starts its gate at the report that says it replaced. Not its output, which // may change on every ask. @@ -1436,6 +1442,26 @@ func adoptionFingerprint(r link.Report) string { return strings.Join(parts, "\n") } +// attachedNetworksOf is the networks a report's profile says the machine is attached to, as one line: the +// part of what the machine is made of that the filter is composed from (novox/hq ADR 0340). +func attachedNetworksOf(r link.Report) string { + machine, ok := r.Profile["machine"].(map[string]any) + if !ok { + return "" + } + networks, ok := machine["networks"].([]any) + if !ok { + return "" + } + said := make([]string, 0, len(networks)) + for _, n := range networks { + if entry, ok := n.(map[string]any); ok { + said = append(said, fmt.Sprintf("%v %v", entry["link"], entry["network"])) + } + } + return strings.Join(said, ",") +} + // applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not // apply and which keys its failure is said under. func applyState(r link.Report) string { diff --git a/examples/README.md b/examples/README.md index 5ad1d1c..a9691ee 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,7 +12,8 @@ mesh runs on: 2 a database per context `inventory`, `identity` and `licences` 3 those contexts' schemas mesh-controller migrate 4 the bus's certificate mesh-controller broker certificate -5 the bus runs NATS, with the controller's account as its first user +5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it + (ADR 0333), with the controller's account as its first user 6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS) ``` @@ -26,6 +27,12 @@ rootsecrets.go). A template whose bus connection it does not recognise is refuse naming the placeholder it looked for. The bundle that raised the previous broker was deleted with ADR 0331; there is one bundle, and every lab bed names it. +**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container +`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in +`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus +port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once, +on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest. + Build an installer carrying a builder, and point it at this: ``` diff --git a/examples/bundle_test.go b/examples/bundle_test.go index 85fec2d..ca3a346 100644 --- a/examples/bundle_test.go +++ b/examples/bundle_test.go @@ -49,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin } sort.Strings(images) - want := []string{"mesh-controller", "nats", "postgres"} + want := []string{"mesh-controller", "nats", "pgvector"} if strings.Join(images, ",") != strings.Join(want, ",") { t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+ "Adding one is a change to what every first node becomes, and to ADR 0006's "+ diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 289e3e2..9f59df6 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -28,11 +28,12 @@ // image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The // registry prefix is the lab's; the digests are upstream's. // -// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host -// creates is owned by root, and the database runs as somebody else inside the container — so it -// could not write, and the container crash-looped. A named volume lets the image set up its own -// ownership, and outlives the container, which is what you want for the thing holding the mesh's -// state. +// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container +// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by +// the image's postgres user, its superuser password read from the module's own-secret file — so the +// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's +// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds +// them to each other against the module's manifest). { "declaration": 1, @@ -83,17 +84,28 @@ "boot": "enabled", "restart-on": ["base-filter"] }, + // The store's data is a directory owned by the image's postgres user (999), made before the + // container starts: a directory the runtime made would be root's, and the database could not + // write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory + // made so (novox/hq ADR 0338 §5). + { + "id": "store-data", + "type": "directory", + "path": "/var/lib/mesh-store", + "mode": "0700", + "owner": "999:70" + }, { "id": "store", "type": "container", - "name": "mesh-store", - "image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a", + "name": "postgres", + "image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f", "env": { "POSTGRES_PASSWORD": "bootstrap", "PGDATA": "/var/lib/postgresql/data/pgdata" }, "ports": ["5432:5432"], - "volumes": ["mesh-store-data:/var/lib/postgresql/data"] + "volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"] }, // Over TCP, not the socket. While the store initialises it runs a temporary server on the // socket ONLY, then stops it and starts the real one — so a socket check passes, the action @@ -102,21 +114,21 @@ { "id": "store-ready", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"], "verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"] }, { "id": "inventory-database", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"], "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"] }, { "id": "identity-database", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"], "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"] }, @@ -126,20 +138,44 @@ { "id": "licences-database", "type": "action", - "in": "mesh-store", + "in": "postgres", "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"], "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"] }, { "id": "context-schemas", "type": "action", - "command": ["docker", "run", "--rm", "--network", "container:mesh-store", + "command": ["docker", "run", "--rm", "--network", "container:postgres", "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "migrate"], - "verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] + "verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] + }, + // **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container + // `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in + // /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the + // catalogue's module declares. So adopting it as that module at the broker step keeps it: the + // module's built image takes the place of the upstream one below, once, on the same streams, users + // and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other + // against the module's own manifest (busLayout), at the broker step and in its tests. + // + // Directories on the machine, not volumes the runtime names: the module mounts these paths, and the + // control plane, once it runs as the mesh's process, reads the certificate at its path on the machine. + // The certificate's directory is readable by everyone because the certificate is public and the control + // plane runs as nobody; its key is written 0600 by the program that makes it. + { + "id": "bus-tls-dir", + "type": "directory", + "path": "/var/lib/mesh-broker-tls", + "mode": "0755" + }, + { + "id": "bus-data-dir", + "type": "directory", + "path": "/var/lib/mesh-broker-nats", + "mode": "0700" }, { "id": "bus-certificate", @@ -151,49 +187,63 @@ // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh // to ask an authority of. - // `--user 0:0` because the volume is root's and this image runs as nobody, which is right - // for the long-running control plane and wrong for a one-shot writing into a fresh volume. - "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", + // `--user 0:0` because the directory is root's and this image runs as nobody, which is right + // for the long-running control plane and wrong for a one-shot writing into a fresh directory. + "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "broker", "certificate", "--into", "/tls"], - "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", + "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "broker", "certificate", "--check", "--into", "/tls"] }, + // **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are + // the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer + // run again after the adoption applies this bundle again, and an ordinary file would put the + // bundle's one user back over every user the mesh composed, and leave the node unable to reach its + // bus. Until the adoption the installer places the composed list over the seed (enrol.go). { "id": "bus-conf-dir", "type": "directory", - "path": "/var/lib/mesh-bus-conf", + "path": "/var/lib/nats-module/conf", "mode": "0700" }, { "id": "bus-conf", "type": "file", - "path": "/var/lib/mesh-bus-conf/nats.conf", + "path": "/var/lib/nats-module/conf/nats.conf", "mode": "0644", - "content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" + "create-once": true, + "content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" }, { "id": "bus-accounts", "type": "file", - "path": "/var/lib/mesh-bus-conf/accounts.conf", + "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0600", + "create-once": true, "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, + // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the + // mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned + // step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17), + // pinned upstream by that same digest: the adoption changes no server release, so it can be undone + // by putting the old image back and is said reversible. The published port is the + // node's bus port, 5671 unless given, which every enrolment token names; the installer gives the + // module the setting that maps its declared 4222 to it (ADR 0333 §2). { "id": "broker", "type": "container", - "name": "mesh-broker", - "image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927", + "name": "mesh-broker-nats", + "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00", "ports": ["5671:4222", "127.0.0.1:8222:8222"], - "volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"], + "volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"], "args": ["-c", "/etc/nats/nats.conf", "-js"] }, { "id": "broker-ready", "type": "action", - "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], - "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"] + "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], + "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"] }, { "id": "control-plane", @@ -202,7 +252,7 @@ "image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "network": "host", "args": ["serve"], - "volumes": ["mesh-broker-tls:/broker-tls:ro"], + "volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"], "env": { "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 21a1dee..dbb2e08 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -174,6 +174,10 @@ type Options struct { // ToolsSource is where the shared base is built from. Everything with code of its own // compiles against it, so it is the first thing the mesh builds for itself. ToolsSource Source + // ToolsGoSource is where the Go toolchain is built from (`mesh-tools-go`, ADR 0142): every Go + // bundle compiles in it, and genesis's own modules have Go code (issue 616). Empty means the + // repository beside ToolsSource named mesh-tools-go, as the mesh keeps it. + ToolsGoSource Source // CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue // CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it. CatalogSource Source @@ -734,7 +738,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro // ---- broker --------------------------------------------------------------------------- say("broker — the foundation's bus, adopted as the nats module: one bus, not two") - if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { + if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, d.Fetch, say); err != nil { return result, failed(StepBroker, err) } diff --git a/internal/bootstrap/builder.go b/internal/bootstrap/builder.go index b86c1b1..3c650c0 100644 --- a/internal/bootstrap/builder.go +++ b/internal/bootstrap/builder.go @@ -1,13 +1,16 @@ package bootstrap import ( + "bytes" "context" + "encoding/json" "fmt" - "strings" ) -// BuilderModule is the module that lets a mesh produce anything at all. -const BuilderModule = "builder" +// BuilderModule is the module that lets a mesh produce anything at all: the build agent, which holds +// the node seat `node-build-agent` (novox/hq ADR 0190). The one-holder `builder` module it replaced +// left the catalogue on 2026-10-03, and genesis went on asking for it (issue 596). +const BuilderModule = "build-agent" // BuilderRepository is what its image is called in this mesh's own registry. const BuilderRepository = "mesh-builder" @@ -50,10 +53,10 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane manifest, err := readManifest(o.Catalogue, BuilderModule) if err != nil { return out, fmt.Errorf("%w\n"+ - "This is the manifest that makes the builder an ordinary module. Without it the mesh "+ + "This is the manifest that makes the build agent an ordinary module. Without it the mesh "+ "has the image and no way to run it, so nothing can be built here", err) } - pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule) + pinned, places, err := pinTheCarriedBuilder(manifest, published.Reference) if err != nil { return out, err } @@ -66,21 +69,72 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane } // Before the push, so the account is in the declaration the machine receives rather than in - // the one after it. A builder that arrives without its credential starts, finds nothing it may - // read, and waits — which looks exactly like a builder with no work. - account := o.Node + "-" + BuilderModule - if _, err := control.tell(ctx, "builder", "issue", account, "--node", o.Node); err != nil { - // Said and carried on. An account that already exists is the ordinary case on a re-run, - // and the push below is what makes either state true on the machine. - if !strings.Contains(err.Error(), "already") { - return out, fmt.Errorf("the builder has no broker account, so it can take no work: %w", err) - } - say(" broker account " + account + " — already issued") - } else { - out.Account = true - say(" broker account " + account + ", scoped to what it consumes and emits") + // the one after it. A build agent that arrives without its credential starts, finds nothing it + // may read, and waits — which looks exactly like an agent with no work. The same act as any + // module's account (`module issue`), which a re-run repeats harmlessly. + if _, err := control.tell(ctx, "module", "issue", BuilderModule, "--node", o.Node); err != nil { + return out, fmt.Errorf("the build agent has no account on the bus, so it can take no work: %w", err) } + out.Account = true + say(" bus account " + BuilderModule + " on " + o.Node + ", scoped to what it consumes and emits") out.Installed.Pushed, err = pushNode(ctx, o, control, say) return out, err } + +// pinTheCarriedBuilder names the image this installer carried and published in the build agent's +// manifest, in place of the artifact the mesh would build for it. +// +// **The one module genesis cannot have built** (novox/hq ADR 0073): the build agent is what builds, +// and at this step the mesh has no build agent. The catalogue's manifest names its server container by +// the artifact it builds (`"artifact": "server"`, an image built from mesh-controller's +// cmd/mesh-builder); the image this installer carries is that build, made from the same program. So +// every container built from that image artifact runs the published reference instead, and the rest +// of the manifest — its build, its seat, its requirements — is the catalogue's as written. The mesh +// rebuilds it from the catalogue like any module from then on. +// +// A manifest still carrying the old placeholder digest is pinned the old way. One naming neither an +// image artifact nor a placeholder is refused: there would be nothing to put the carried image in. +func pinTheCarriedBuilder(manifest []byte, reference string) ([]byte, int, error) { + if bytes.Contains(manifest, []byte(placeholderDigest)) { + return pinPlaceholder(manifest, reference, BuilderModule) + } + var m map[string]any + if err := json.Unmarshal(manifest, &m); err != nil { + return nil, 0, fmt.Errorf("the %s module's manifest is not readable: %w", BuilderModule, err) + } + images := map[string]bool{} + if build, ok := m["build"].(map[string]any); ok { + artifacts, _ := build["artifacts"].([]any) + for _, a := range artifacts { + if art, ok := a.(map[string]any); ok && art["kind"] == "image" { + if name, ok := art["name"].(string); ok { + images[name] = true + } + } + } + } + places := 0 + resources, _ := m["resources"].([]any) + for _, r := range resources { + res, ok := r.(map[string]any) + if !ok || res["type"] != "container" { + continue + } + if name, ok := res["artifact"].(string); ok && images[name] { + delete(res, "artifact") + res["image"] = reference + places++ + } + } + if places == 0 { + return nil, 0, fmt.Errorf( + "the %s module's manifest runs no container from an image it builds, so there is nowhere to "+ + "put the builder this installer carried and published (%s)", BuilderModule, reference) + } + pinned, err := json.MarshalIndent(m, "", " ") + if err != nil { + return nil, 0, err + } + return append(pinned, '\n'), places, nil +} diff --git a/internal/bootstrap/builder_test.go b/internal/bootstrap/builder_test.go index 26368e9..f4158f1 100644 --- a/internal/bootstrap/builder_test.go +++ b/internal/bootstrap/builder_test.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "encoding/json" "os" "path/filepath" "strings" @@ -37,27 +38,6 @@ func recording(t *testing.T) (*controlRecorder, controlPlane) { return c, controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second} } -func TestTheBuildersPackageRegistryPortIsASettingAndNotTheManifest(t *testing.T) { - c, control := recording(t) - o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second} - if _, err := installModule(context.Background(), o, control, BuilderModule, - []byte(theBuildersBinding), quietly); err != nil { - t.Fatal(err) - } - if got := c.settings["builder-settings.json"]; got != `{"serves":{"port":3100}}` { - t.Errorf("the builder was told %q about the package registry's port", got) - } - // The manifest reaches the mesh as the catalogue wrote it. A port rewritten into it here is a - // port the next registration from the catalogue silently takes back. - if got := c.settings["builder-module.json"]; got != theBuildersBinding { - t.Errorf("the installer changed the builder's manifest:\n%s", got) - } - set, push := c.index("settings set builder"), c.index("push anchor") - if set < 0 || push < 0 || set > push { - t.Fatalf("the port was not set before the push that raises the builder: %v", c.told) - } -} - func TestTheBuilderOnTheDefaultPackagesPortIsToldNothing(t *testing.T) { c, control := recording(t) o := Options{Node: "anchor", Wait: time.Second} @@ -167,3 +147,73 @@ func TestAForgeManifestTheCatalogueDoesNotHaveIsNamed(t *testing.T) { t.Errorf("a missing forge manifest was not explained: %v", err) } } + +// theCataloguesModule is the catalogue's own manifest of a module, never a copy: the clone beside a merge +// check (MESH_CHECK_BESIDE), where it must be; else a checkout beside this one; else the test skips aloud. +func theCataloguesModule(t *testing.T, module string) []byte { + t.Helper() + inside := filepath.Join("mesh-catalog", "modules", module, "module.json") + if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" { + raw, err := os.ReadFile(filepath.Join(root, inside)) + if err != nil { + t.Fatalf("the catalogue beside this check (MESH_CHECK_BESIDE=%s) has no %s: %v", root, inside, err) + } + return raw + } + raw, err := os.ReadFile(filepath.Join("..", "..", "..", inside)) + if err != nil { + t.Skipf("NOT JUDGED: no catalogue beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it") + } + return raw +} + +// **Genesis installs the build agent the catalogue holds** (novox/hq issue 596, ADR 0190): the +// one-holder `builder` left the catalogue on 2026-10-03, and genesis stopped at step 13 asking for it. +// The catalogue's own manifest is read, and the image the installer carried is put where the manifest +// names the image it builds. +func TestGenesisInstallsTheCataloguesBuildAgent(t *testing.T) { + manifest := theCataloguesModule(t, BuilderModule) + const reference = "127.0.0.1:5000/mesh-builder@sha256:" + "4444444444444444444444444444444444444444444444444444444444444444" + pinned, places, err := pinTheCarriedBuilder(manifest, reference) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Claims []struct { + Name string `json:"name"` + } `json:"claims"` + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal(pinned, &m); err != nil { + t.Fatal(err) + } + if m.Module != BuilderModule || len(m.Claims) == 0 || m.Claims[0].Name != "node-build-agent" { + t.Fatalf("genesis installs %q claiming %v; the build agent holds node-build-agent (ADR 0190)", m.Module, m.Claims) + } + running := 0 + for _, r := range m.Resources { + if r["type"] != "container" { + continue + } + if _, still := r["artifact"]; still { + t.Errorf("the container %v still names an artifact nothing has built yet", r["name"]) + } + if r["image"] == reference { + running++ + } + } + if places != 1 || running != 1 { + t.Errorf("the carried builder runs in %d container(s) (%d place(s) said); the build agent runs one", running, places) + } +} + +// A manifest that runs nothing from an image it builds has nowhere to put the carried builder, and is +// refused rather than registered running nothing. +func TestABuildAgentManifestWithNowhereForTheCarriedBuilderIsRefused(t *testing.T) { + _, _, err := pinTheCarriedBuilder([]byte(`{"module":"build-agent","resources":[{"id":"server","type":"container","name":"x","artifact":"server"}],"build":{"artifacts":[{"name":"server","kind":"bundle"}]}}`), + "127.0.0.1:5000/mesh-builder@sha256:"+strings.Repeat("4", 64)) + if err == nil || !strings.Contains(err.Error(), "nowhere to put") { + t.Fatalf("a manifest with no image artifact was pinned: %v", err) + } +} diff --git a/internal/bootstrap/bus_adoption.go b/internal/bootstrap/bus_adoption.go new file mode 100644 index 0000000..4f26ea0 --- /dev/null +++ b/internal/bootstrap/bus_adoption.go @@ -0,0 +1,179 @@ +package bootstrap + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Adopting the bus is the mesh's planned step for replacing it (novox/hq ADR 0333 §3, ADR 0236). +// +// The adoption replaces the bus's image once: the module's built image takes the place of the upstream +// one the bundle raised, on the same streams, users and certificate. The controller refuses a push that +// would replace the bus — "a planned step: `bus upgrade --why …` snapshots its streams first and checks +// them after" — so genesis stopped at step 18 (issue 590). The installer takes that step: it snapshots the +// streams itself, because at genesis no backup holder exists to be asked, and hands the snapshot to +// `bus upgrade`, which sends the machine, opens `bus-maintenance`, and ends the step done only once the +// bus answers healthy again — every stream, every durable consumer, a round trip (H-bus). The installer +// waits for that end. + +// adoptionWhy is what the step is recorded as, for the record of hand acts and the bus's step. +const adoptionWhy = "genesis adopts its bus as the nats module (ADR 0333)" + +// Where the installer keeps the streams' snapshot before the adoption, and the credential the snapshot +// program reads while it runs. Beside genesis's own credentials; variables so a test writes elsewhere. +var ( + busSnapshotFile = "/var/lib/mesh-genesis/bus-before-adoption.tar" + busSnapshotCredential = "/var/lib/mesh-genesis/bus-snapshot.credential" + busCertificateFile = "/var/lib/mesh-broker-tls/tls.crt" + busControllerSecret = BusControllerFile + // busStepWithin bounds the wait for the controller to say the step ended: its own bound (15 min, + // mesh-controller busStepBound), and its doctor's next look after that. + busStepWithin = 17 * time.Minute +) + +// adoptTheBus sends the nats module to this machine through the bus's planned step. A bus already on the +// module's image is the module's: it is sent like any module, and nothing is replaced. +func adoptTheBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container, + say func(string)) error { + + if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image { + say(" already adopted " + bus.Name + " — it runs the module's image") + _, err := pushNode(ctx, o, control, say) + return err + } + image, err := theModulesBusImage(ctx, o, control) + if err != nil { + return err + } + if err := snapshotTheBus(ctx, o, control, image, say); err != nil { + return err + } + say(" restarting " + bus.Name + " — once, onto the " + BusModule + " module's image, as the bus's " + + "planned step; its streams, users and certificate stay where they are (novox/hq ADR 0333 §3, ADR 0236)") + if _, err := control.within(2*o.Wait).tell(ctx, "bus", "upgrade", "--why", adoptionWhy, + "--reversible", "--snapshot-taken", busSnapshotFile); err != nil { + return fmt.Errorf("the bus's planned step was refused, so the bus is not replaced: %w", err) + } + return nil +} + +// meshBuiltImage is an image the mesh built, as it records it: `artifact-store:///@sha256:…`. +var meshBuiltImage = regexp.MustCompile(`artifact-store://([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`) + +// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this +// mesh's registry: the image that carries the snapshot program. +func theModulesBusImage(ctx context.Context, o Options, control controlPlane) (string, error) { + said, err := control.tell(ctx, "builds", BusModule, "-n", "1") + if err != nil { + return "", err + } + for _, m := range meshBuiltImage.FindAllStringSubmatch(said, -1) { + if strings.HasPrefix(m[1], BusModule+"/") { + return strings.TrimSuffix(o.Registry, "/") + "/" + m[1], nil + } + } + return "", fmt.Errorf("the mesh says no image it built for the %s module, so its snapshot program cannot "+ + "be run before the bus is replaced:\n%s", BusModule, indent(strings.TrimSpace(said))) +} + +// snapshotTheBus writes every stream of the running bus to busSnapshotFile with the module's own snapshot +// program (mesh-catalog modules/nats, ADR 0235), as the controller's account, pinned to the bus's +// certificate. The credential is written for the program's one run and removed after it. +func snapshotTheBus(ctx context.Context, o Options, control controlPlane, image string, say func(string)) error { + password, err := readCredentialFile(busControllerSecret) + if err != nil { + return fmt.Errorf("the controller's bus credential is not at %s, so the streams cannot be snapshotted "+ + "before the bus is replaced: %w", busControllerSecret, err) + } + fingerprint, err := certificateFingerprint(busCertificateFile) + if err != nil { + return err + } + port := o.Ports.orDefaults().Bus + server := "tls://127.0.0.1:" + strconv.Itoa(port) + credential, err := json.Marshal(map[string]string{"url": server, "fingerprint": fingerprint, + "user": busControllerUser, "password": password}) + if err != nil { + return err + } + if err := os.WriteFile(busSnapshotCredential, credential, 0o600); err != nil { + return fmt.Errorf("the snapshot's credential could not be written: %w", err) + } + defer os.Remove(busSnapshotCredential) + + script := "docker run --rm -i --network host --entrypoint /usr/local/bin/mesh-nats-snapshot " + image + + " snapshot --server " + server + " --credential - < " + busSnapshotCredential + " > " + busSnapshotFile + + ".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile + running, cancel := context.WithTimeout(ctx, busStepWithin) + defer cancel() + if out, err := control.run(running, "sh", "-c", script); err != nil { + _, _ = control.run(ctx, "rm", "-f", busSnapshotFile+".partial") + return fmt.Errorf("the bus's streams could not be snapshotted, so the bus is not replaced: %w\n%s", + err, indent(strings.TrimSpace(out))) + } + say(" snapshotted every stream of the bus, at " + busSnapshotFile + " (its module's snapshot program)") + return nil +} + +// certificateFingerprint is the pin of the bus's certificate: sha256 of its DER, as every machine pins it. +func certificateFingerprint(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("the bus's certificate is not at %s, so it cannot be pinned for the snapshot: %w", path, err) + } + block, _ := pem.Decode(raw) + if block == nil { + return "", fmt.Errorf("%s holds no PEM certificate", path) + } + sum := sha256.Sum256(block.Bytes) + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +// waitForTheBusStep waits until the controller says the bus's planned step ended: done once the bus +// answered healthy — every stream, every durable consumer, a round trip — or failed, with its snapshot as +// the way back. The step's own bound is the controller's; this waits a little past it. +func waitForTheBusStep(ctx context.Context, o Options, control controlPlane, say func(string)) error { + deadline := time.Now().Add(busStepWithin) + var last string + for { + said, err := control.tell(ctx, "bus") + if err == nil { + for _, line := range strings.Split(said, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "last step ") { + last = strings.TrimSpace(line) + } + } + switch { + case strings.Contains(last, ": done at "): + say(" bus step done " + last) + return nil + case strings.Contains(last, ": failed at "): + return fmt.Errorf("the bus's planned step ended failed: %s\n%s", last, indent(strings.TrimSpace(said))) + } + } + if !time.Now().Before(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + if last == "" { + return errors.New("the controller says no bus step after the adoption was sent; `mesh-controller bus` says how it stands") + } + return fmt.Errorf("the bus's planned step did not end within %s: %s", busStepWithin, last) +} diff --git a/internal/bootstrap/bus_adoption_test.go b/internal/bootstrap/bus_adoption_test.go new file mode 100644 index 0000000..cbb52e9 --- /dev/null +++ b/internal/bootstrap/bus_adoption_test.go @@ -0,0 +1,135 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" +) + +// Defends novox/hq issue 590 at its broker step (ADR 0333 §3, ADR 0236): the adoption replaces the bus's +// image, and the mesh replaces its bus only as a planned step. A plain push was refused — "sending anchor +// would replace the bus … a planned step: `bus upgrade --why …`" — and genesis stopped at step 18. + +func TestTheBrokerStepAdoptsTheBusThroughItsPlannedStep(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image + upgraded := false + var credential []byte + runtime := &asked{} + agrees := aMeshThatAgrees(nil) + runtime.answer = func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case name == "sh" && strings.Contains(joined, "mesh-nats-snapshot"): + credential, _ = os.ReadFile(busSnapshotCredential) // as the program reads it + case strings.Contains(joined, "bus upgrade --why"): + upgraded = true + case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"): + if upgraded { + return theModulesImage + "\n", nil + } + return bundleImage + "\n", nil + } + return agrees(name, args) + } + _, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Wait = 5 * time.Second + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + + snapshot, upgrade := -1, -1 + for i, c := range runtime.commands { + switch { + case strings.HasPrefix(c, "sh -c") && strings.Contains(c, "mesh-nats-snapshot"): + snapshot = i + for _, want := range []string{"127.0.0.1:5000/nats/server@sha256:" + strings.Repeat("4", 64), + "snapshot --server tls://127.0.0.1:5671 --credential - < " + busSnapshotCredential, + "> " + busSnapshotFile + ".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile} { + if !strings.Contains(c, want) { + t.Errorf("the snapshot is not taken with %q: %s", want, c) + } + } + case strings.Contains(c, "/mesh-controller bus upgrade"): + upgrade = i + if want := "bus upgrade --why " + adoptionWhy + " --reversible --snapshot-taken " + busSnapshotFile; !strings.HasSuffix(c, want) { + t.Errorf("the planned step is asked as %q; want it to end %q", c, want) + } + case strings.HasSuffix(c, "/mesh-controller push anchor"): + t.Errorf("the bus's module was sent by a plain push, which the mesh refuses for the bus: %v", runtime.commands) + } + } + if snapshot < 0 || upgrade < 0 || snapshot > upgrade { + t.Fatalf("the streams were not snapshotted before the bus's planned step (snapshot %d, upgrade %d): %v", + snapshot, upgrade, runtime.commands) + } + var cred map[string]string + if err := json.Unmarshal(credential, &cred); err != nil { + t.Fatalf("the snapshot program was given no credential: %v", err) + } + fingerprint, _ := certificateFingerprint(busCertificateFile) + if cred["user"] != "controller" || cred["password"] != "BUS-PW" || cred["fingerprint"] != fingerprint || + cred["url"] != "tls://127.0.0.1:5671" { + t.Errorf("the snapshot's credential is %v", cred) + } + if _, err := os.Stat(busSnapshotCredential); !os.IsNotExist(err) { + t.Error("the snapshot's credential was left on the machine") + } + text := strings.Join(said, "\n") + for _, want := range []string{"snapshotted every stream", "restarting mesh-broker-nats — once", + "bus answers mesh-broker-nats", "bus step done last step 1", "adopted mesh-broker-nats"} { + if !strings.Contains(text, want) { + t.Errorf("the step did not say %q:\n%s", want, text) + } + } +} + +// A planned step the controller ends failed is the broker step failing, with the controller's words. +func TestABusStepThatEndsFailedIsRefused(t *testing.T) { + aGenesisBusOnDisk(t) + runtime := &asked{answer: func(name string, args []string) (string, error) { + return "last step 1: not known → 53e51864 on anchor by installer (x): failed at 2026-10-13 11:59; snapshot s\n" + + " stream KV_x has no messages\n", nil + }} + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} + err := waitForTheBusStep(context.Background(), Options{}, control, quietly) + if err == nil || !strings.Contains(err.Error(), "failed at") || !strings.Contains(err.Error(), "KV_x") { + t.Fatalf("a failed bus step was not refused with its words: %v", err) + } +} + +// The bundle's bus runs the release the module's image is built on, pinned by the same digest, so the +// adoption changes no server release and is reversible as the step says (ADR 0333 §3). +func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) { + var m struct { + Build struct { + On []struct{ Arg, Image string } `json:"on"` + } `json:"build"` + } + if err := json.Unmarshal(theCataloguesModule(t, BusModule), &m); err != nil { + t.Fatal(err) + } + base := "" + for _, on := range m.Build.On { + if on.Arg == "NATS_BASE" { + base = on.Image + } + } + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if got := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image; got != base { + t.Errorf("the bundle raises the bus on %s; the nats module's image is built on %s", got, base) + } +} diff --git a/internal/bootstrap/control.go b/internal/bootstrap/control.go index bcade8b..a0c509b 100644 --- a/internal/bootstrap/control.go +++ b/internal/bootstrap/control.go @@ -311,6 +311,17 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) { for name, path := range m.OwnSecrets { secretAt[path] = name } + // **A file the module itself declares is its own, not a secret** (novox/hq issue 597). The + // controller reads its merge window's settings from a file its manifest writes from its settings + // (MESH_MERGE_WINDOW_FILE, ADR 0276): the mesh writes it with the module's declaration, and there + // is nothing for genesis to carry in. Only a `…_FILE` naming neither an own-secret nor a file of + // the module is refused. + declared := map[string]bool{} + for _, r := range m.Resources { + if r.Type == "file" && r.Path != "" { + declared[r.Path] = true + } + } wanted := map[string]string{} for _, r := range m.Resources { @@ -333,10 +344,13 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) { on = from } secret, named := secretAt[on] + if !named && declared[on] { + continue + } if !named { return nil, fmt.Errorf( - "the %s module's container reads %s from %s, and no own-secret of that "+ - "module writes that file.\n"+ + "the %s module's container reads %s from %s, and neither an own-secret nor "+ + "a file of that module writes that file.\n"+ "So the mesh would seal nothing there and the control plane would find "+ "an empty file where a connection string has to be. The manifest has to "+ "name the two ends the same, directly or through a mount", diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index b0d13d2..89e1228 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -94,10 +94,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla switch mine, err := identity.Load(where); { case err == nil && mine.Node == o.Node: say(" already enrolled " + o.Node + " — " + where) - // **Enrolled is not adopted** (novox/hq ADR 0331 §5). This run's apply wrote the bundle's - // user list again — the controller's entry and nobody else's — over the list placed when - // the machine enrolled. Until the bus is the nats module, that list is still the - // installer's to place, so it is placed again, before anything dials the bus as a node. + // **Enrolled is not adopted** (novox/hq ADR 0331 §5). Until the bus is the nats module, its + // user list is the installer's to place, so it is placed again, before anything dials the + // bus as a node: the controller's composition is the one that is right, whatever an earlier + // run left there. The bundle seeds the list only where none is (create-once, ADR 0333 §4). if err := placeUntilAdopted(ctx, control, say); err != nil { return out, err } @@ -168,9 +168,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla // configuration includes, and the container that reads it. The installer raised them, so it is the // one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the // nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5). +// +// **The module's path and the module's container** (novox/hq ADR 0333 §4): the bundle raises the bus +// where the `nats` module keeps it, so the list placed here is the one the module goes on writing, in +// the same file, read by the same container. The tests hold both to the bundle and to the module's +// manifest (`bus-users` and its server container). const ( - busAccounts = "/var/lib/mesh-bus-conf/accounts.conf" - busContainer = "mesh-broker" + busAccounts = "/var/lib/nats-module/conf/accounts.conf" + busContainer = "mesh-broker-nats" ) // placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes diff --git a/internal/bootstrap/enrol_test.go b/internal/bootstrap/enrol_test.go index 111d000..031b823 100644 --- a/internal/bootstrap/enrol_test.go +++ b/internal/bootstrap/enrol_test.go @@ -315,8 +315,8 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) { placed, enrol := []int{}, -1 for i, c := range runtime.commands { - if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") && - strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") { + if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker-nats") && + strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") { placed = append(placed, i) } if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") { @@ -365,7 +365,7 @@ func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *tes placed := false for _, c := range runtime.commands { if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") && - strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker") { + strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker-nats") { placed = true } } diff --git a/internal/bootstrap/genesis_form_test.go b/internal/bootstrap/genesis_form_test.go index e009568..b22da3a 100644 --- a/internal/bootstrap/genesis_form_test.go +++ b/internal/bootstrap/genesis_form_test.go @@ -180,3 +180,85 @@ func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) { } } } + +// theStepNineSecretsOf is what step 9 delivers for a process-form controller manifest: its genesis +// container made, pinned, and its secrets read from that container's environment. +func theStepNineSecretsOf(t *testing.T, manifest []byte) (map[string]string, error) { + t.Helper() + form, err := processFormOf(manifest) + if err != nil || !form.Found { + t.Fatalf("the manifest has no process form: %v", err) + } + genesis, err := genesisForm(manifest, form, builtImage) + if err != nil { + t.Fatal(err) + } + pinned, _, err := pinImage(genesis, builtImage, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule) + if err != nil { + t.Fatal(err) + } + return secretsByVariableIn(pinned) +} + +// **A file the controller's manifest writes itself is not a secret** (novox/hq issue 597). The merge +// window's settings reach the controller in a file its manifest declares from its settings +// (MESH_MERGE_WINDOW_FILE, ADR 0276); genesis refused it as a secret nobody writes, and stopped at +// step 11 of 24. +func TestAFileTheControllersManifestWritesIsNotASecret(t *testing.T) { + manifest := strings.Replace(processFormManifest, + `{"id": "controller", "type": "process",`, + `{"id": "merge-window", "type": "file", "path": "${dir:mesh-state}/merge-window.json", "mode": "0600", + "owner": "mesh-controller", "content": "{\"merge-window\": \"${setting:merge-window}\"}\n"}, + {"id": "controller", "type": "process",`, 1) + manifest = strings.Replace(manifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`, + `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus", + "MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1) + if !strings.Contains(manifest, "MESH_MERGE_WINDOW_FILE") || !strings.Contains(manifest, `"id": "merge-window"`) { + t.Fatal("the fixture was not changed") + } + wanted, err := theStepNineSecretsOf(t, []byte(manifest)) + if err != nil { + t.Fatalf("step 9 refuses a file the manifest writes itself: %v", err) + } + if _, asked := wanted["MESH_MERGE_WINDOW"]; asked { + t.Errorf("the merge window's settings are delivered as a secret: %v", wanted) + } + if wanted["MESH_STORE_INVENTORY"] != "inventory" || wanted["MESH_BUS_NATS"] != "bus" { + t.Errorf("step 9 delivers %v; the store's and the bus's secrets are still wanted", wanted) + } +} + +// And a `…_FILE` that neither an own-secret nor a file of the module writes is still refused. +func TestAFileNothingOfTheControllerWritesIsStillRefused(t *testing.T) { + manifest := strings.Replace(processFormManifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`, + `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus", + "MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1) + if _, err := theStepNineSecretsOf(t, []byte(manifest)); err == nil || !strings.Contains(err.Error(), "MESH_MERGE_WINDOW_FILE") { + t.Fatalf("a file nothing writes was accepted, or refused without its name: %v", err) + } +} + +// The controller's own manifest, as the mesh runs it, passes step 9 (issue 597): in a merge check the +// node-engine's check has the controller cloned beside it (MESH_CHECK_BESIDE), where it must be; else a +// checkout beside this one; else it skips aloud. +func TestTheControllersOwnManifestPassesStepNine(t *testing.T) { + var manifest []byte + if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" { + raw, err := os.ReadFile(filepath.Join(root, "mesh-controller", "module.json")) + if err != nil { + t.Fatalf("the controller is not beside this check in MESH_CHECK_BESIDE=%s: %v", root, err) + } + manifest = raw + } else if raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-controller", "module.json")); err == nil { + manifest = raw + } else { + t.Skip("NOT JUDGED: no controller beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it") + } + wanted, err := theStepNineSecretsOf(t, manifest) + if err != nil { + t.Fatalf("genesis's step 9 refuses the controller the mesh runs: %v", err) + } + if wanted["MESH_STORE_INVENTORY"] != "inventory" { + t.Errorf("step 9 delivers %v", wanted) + } +} diff --git a/internal/bootstrap/genesis_registry_test.go b/internal/bootstrap/genesis_registry_test.go new file mode 100644 index 0000000..7cd3448 --- /dev/null +++ b/internal/bootstrap/genesis_registry_test.go @@ -0,0 +1,65 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +// Genesis seals the build agent the forge it raised (novox/hq ADR 0336 §3, issue 121): the agent's +// npm-package-registry requirement is optional and binds only once the gitea module is assigned, after +// the base, so until then the packages step's genesis document is how the sdk and base steps reach the +// forge. + +func TestThePackagesStepSealsTheGenesisDocumentToTheBuildAgent(t *testing.T) { + if BuilderModule != "build-agent" { + t.Fatalf("genesis installs %q; the document is the build agent's", BuilderModule) + } + c, control := recording(t) + o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second} + doc := genesisRegistry(o.Ports.orDefaults(), "builder-password") + if err := deliverGenesisRegistry(context.Background(), o, control, doc, quietly); err != nil { + t.Fatal(err) + } + carried := c.settings["accepting-"+GenesisRegistrySecret] + var got GenesisRegistry + if err := json.Unmarshal([]byte(carried), &got); err != nil { + t.Fatalf("the genesis document carried is not JSON: %q", carried) + } + want := GenesisRegistry{Scheme: "http", At: "127.0.0.1", Port: 3100, NpmPath: "/api/packages/novox/npm/", + As: "mesh-builder", Password: "builder-password"} + if got != want { + t.Fatalf("the genesis document says %+v, want %+v", got, want) + } + accept := c.index("secret accept anchor build-agent genesis-package-registry --from /accepting-genesis-package-registry") + push := c.index("push anchor") + if accept < 0 || push < 0 || accept > push { + t.Fatalf("the genesis document was not sealed to the build agent before the push: %v", c.told) + } +} + +// What the retired builder was given and the build agent declares neither of: an `npm-password` +// own-secret and a `serves` setting for its carried binding (ADR 0336 §3). +func TestNoStepGivesTheBuildAgentAServesSettingOrAnNpmPassword(t *testing.T) { + for _, module := range []string{BuilderModule} { + c, control := recording(t) + o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second} + if _, err := installModule(context.Background(), o, control, module, []byte(`{"module":"`+module+`"}`), quietly); err != nil { + t.Fatal(err) + } + if got, set := c.settings[module+"-settings.json"]; set { + t.Errorf("%s was given a setting on a node with a non-default packages port: %s", module, got) + } + if err := deliverGenesisRegistry(context.Background(), o, control, + genesisRegistry(o.Ports.orDefaults(), "p"), quietly); err != nil { + t.Fatal(err) + } + for _, told := range c.told { + if strings.Contains(told, "npm-password") || strings.Contains(told, "serves") { + t.Errorf("genesis told the controller %q", told) + } + } + } +} diff --git a/internal/bootstrap/genesis_toolchains_test.go b/internal/bootstrap/genesis_toolchains_test.go new file mode 100644 index 0000000..8523054 --- /dev/null +++ b/internal/bootstrap/genesis_toolchains_test.go @@ -0,0 +1,89 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +// Defends novox/hq issue 616: genesis's base step builds the toolchain of every language a module +// genesis installs is compiled in. It built only mesh-tools, and the store's module, with Go code, +// stopped genesis at step 17: "this mesh holds no copy of it. Build mesh-tools-go first". + +// genesisModules is every catalogue module genesis installs on its own, read from the steps that +// install them: the registry, the build agent, the store, the bus, the vault, the catalogue and the +// filter (extras are the operator's choice and come after the base too, but are not genesis's own). +var genesisModules = []string{RegistryModule, BuilderModule, "postgres", BusModule, "mesh-vault", "mesh-catalog", "nftables"} + +// Every genesis module's bundles are compiled in a toolchain the base step builds, read from the real +// catalogue's manifests: a module that gains a language genesis does not build fails here, not at a +// genesis. +func TestTheBaseStepBuildsEveryToolchainAGenesisModuleIsCompiledIn(t *testing.T) { + built := map[string]bool{} + for _, tc := range genesisToolchains { + built[tc.Language] = true + } + for _, module := range genesisModules { + var m struct { + Build struct { + Artifacts []struct { + Name, Kind, Language string + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(theCataloguesModule(t, module), &m); err != nil { + t.Fatalf("%s: %v", module, err) + } + for _, a := range m.Build.Artifacts { + if a.Kind != "bundle" { + continue // an image builds on the bases its Dockerfile names, pinned upstream + } + if !built[a.Language] { + t.Errorf("%s's %q is a %s bundle, and genesis's base step builds no %s toolchain", + module, a.Name, a.Language, a.Language) + } + } + } +} + +// And the base step asks for each toolchain's build: the base, then the Go toolchain beside it. +func TestTheBaseStepBuildsTheGoToolchainBesideTheBase(t *testing.T) { + runtime := &asked{answer: aMeshThatAgrees(nil)} + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} + o := Options{ToolsSource: Source{Repository: "git://192.0.2.30/mesh-tools.git"}} + if err := BuildBase(context.Background(), o, control, quietly); err != nil { + t.Fatal(err) + } + var builds []string + for _, c := range runtime.commands { + if strings.Contains(c, "/mesh-controller build ") { + builds = append(builds, c[strings.Index(c, "build "):]) + } + } + want := []string{ + "build git://192.0.2.30/mesh-tools.git --ref main --wait 1200s", + "build git://192.0.2.30/mesh-tools-go.git --ref main --wait 1200s", + } + if strings.Join(builds, "\n") != strings.Join(want, "\n") { + t.Fatalf("the base step built\n%s\nand not\n%s", strings.Join(builds, "\n"), strings.Join(want, "\n")) + } +} + +func TestTheGoToolchainIsAsGivenOrBesideTheBase(t *testing.T) { + for _, c := range []struct{ tools, given, want string }{ + {"http://forge/novox/mesh-tools.git", "", "http://forge/novox/mesh-tools-go.git"}, + {"http://forge/novox/mesh-tools", "", "http://forge/novox/mesh-tools-go"}, + {"http://forge/other/base.git", "http://forge/other/go.git", "http://forge/other/go.git"}, + } { + got, err := toolsGoSource(Options{ToolsSource: Source{Repository: c.tools}, ToolsGoSource: Source{Repository: c.given}}) + if err != nil || got.Repository != c.want { + t.Errorf("tools %s, given %q: %q %v; want %s", c.tools, c.given, got.Repository, err, c.want) + } + } + if _, err := toolsGoSource(Options{ToolsSource: Source{Repository: "http://forge/other/base.git"}}); err == nil || + !strings.Contains(err.Error(), "--tools-go-source") { + t.Errorf("a base not named mesh-tools, and no Go toolchain given, was not refused naming the flag: %v", err) + } +} diff --git a/internal/bootstrap/module.go b/internal/bootstrap/module.go index 6352d4d..6b46dd0 100644 --- a/internal/bootstrap/module.go +++ b/internal/bootstrap/module.go @@ -137,7 +137,26 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod // should hold, seals every secret in them and publishes them, and on a first node that is the // slowest thing the mesh does. func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { - said, err := control.within(o.Wait).tell(ctx, "push", o.Node) + return pushing(ctx, o, control, false, say) +} + +// pushNodeApplied is pushNode that returns only once the node reports it applied exactly what it was +// sent (`push --wait`, novox/hq ADR 0010): for a push whose content the very next step reads +// on the machine. A plain push returns as soon as the declaration is sent, and the node applies it a +// moment later (issue 615). +func pushNodeApplied(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { + return pushing(ctx, o, control, true, say) +} + +func pushing(ctx context.Context, o Options, control controlPlane, applied bool, say func(string)) (string, error) { + args := []string{"push", o.Node} + patience := o.Wait + if applied { + args = append(args, "--wait", o.Wait.String()) + // The controller's own wait, and time to compose and send before it starts. + patience = 2 * o.Wait + } + said, err := control.within(patience).tell(ctx, args...) if err != nil { return "", fmt.Errorf( "%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+ @@ -145,7 +164,20 @@ func pushNode(ctx context.Context, o Options, control controlPlane, say func(str "at all. Fix what it named and run this installer again: it will find the module "+ "already registered and try the push again", err) } - say(" pushed " + o.Node) + if applied { + say(" pushed " + o.Node + ", and it applied it") + } else { + say(" pushed " + o.Node) + } + // **Every push may carry a new bus user, and until the adoption the bus hears of it only from here** + // (novox/hq issue 608, ADR 0331 §5, ADR 0333 §4). A module's account is minted into the mesh's + // records and sealed to this node with the push; the bus reads its user list from a file, and until + // the broker step adopts the bus as the nats module nothing but this installer writes that file. The + // list was placed only when the node enrolled, so the build agent's account (step 13) never reached + // the bus and the agent was refused ("Authorization Violation") until the sdk step gave up. + if err := placeUntilAdopted(ctx, control, say); err != nil { + return "", err + } return strings.TrimSpace(said), nil } diff --git a/internal/bootstrap/phase2.go b/internal/bootstrap/phase2.go index 615191a..e49eb1b 100644 --- a/internal/bootstrap/phase2.go +++ b/internal/bootstrap/phase2.go @@ -25,20 +25,62 @@ import ( // buildWait bounds one module build. Generous, because the first build compiles a toolchain. const buildWait = 20 * time.Minute -// BuildBase asks the mesh to build the shared base every module with code of its own stands on. +// BuildBase asks the mesh to build the shared base every module with code of its own stands on, and +// the toolchain of every language genesis's own modules are written in. // // **First, because until it exists nothing else with code can be built.** Not registered as a // module here: it is never assigned — it runs nowhere — and the build itself records what was // made, which is all anything downstream reads. +// +// **Every toolchain a genesis module compiles in** (novox/hq issue 616). A bundle is compiled in its +// language's base (mesh-controller internal/builder toolchain.go): TypeScript in mesh-tools, Go in +// mesh-tools-go. The store's and the bus's modules carry Go code, so a genesis that built only +// mesh-tools stopped at the store step: "this mesh holds no copy of it. Build mesh-tools-go first". +// The bases are the ones genesisToolchains names, and a test holds them to the languages of every +// module genesis installs, read from the catalogue. func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error { if o.ToolsSource.Repository == "" { return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " + "own repository, and an installer told nothing cannot know where that is") } - say(" building " + o.ToolsSource.Repository + " at " + refOr(o.ToolsSource.Ref)) - _, err := control.within(buildWait).tell(ctx, - "build", o.ToolsSource.Repository, "--ref", refOr(o.ToolsSource.Ref), "--wait", "1200s") - return err + goSource, err := toolsGoSource(o) + if err != nil { + return err + } + for _, src := range []Source{o.ToolsSource, goSource} { + say(" building " + src.Repository + " at " + refOr(src.Ref)) + if _, err := control.within(buildWait).tell(ctx, + "build", src.Repository, "--ref", refOr(src.Ref), "--wait", "1200s"); err != nil { + return err + } + } + return nil +} + +// genesisToolchains is the base each language's bundles compile in, for the languages genesis's own +// modules are written in — the same pairs as mesh-controller internal/builder toolchain.go — in the +// order BuildBase builds them. +var genesisToolchains = []struct{ Language, Base string }{ + {"typescript", "mesh-tools"}, + {"go", "mesh-tools-go"}, +} + +// toolsGoSource is where the Go toolchain is built from: as given, or the repository beside the base's +// named mesh-tools-go, which is how the mesh keeps its repositories side by side. +func toolsGoSource(o Options) (Source, error) { + if o.ToolsGoSource.Repository != "" { + return o.ToolsGoSource, nil + } + repo := strings.TrimSuffix(o.ToolsSource.Repository, "/") + suffix := "" + if strings.HasSuffix(repo, ".git") { + repo, suffix = strings.TrimSuffix(repo, ".git"), ".git" + } + if !strings.HasSuffix(repo, "mesh-tools") { + return Source{}, fmt.Errorf("the Go toolchain is built from mesh-tools-go, and --tools-source (%s) is not a "+ + "repository named mesh-tools to find it beside; give --tools-go-source", o.ToolsSource.Repository) + } + return Source{Repository: repo + "-go" + suffix, Ref: o.ToolsGoSource.Ref}, nil } // InstallFromCatalogue builds a catalogue module and installs it on this machine. diff --git a/internal/bootstrap/phase3.go b/internal/bootstrap/phase3.go index c8aadbc..ae45e01 100644 --- a/internal/bootstrap/phase3.go +++ b/internal/bootstrap/phase3.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "fmt" + "net/http" "strings" + "time" "github.com/novox/mesh-host/internal/declaration" ) @@ -82,7 +84,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, say); err != nil { return err } - say(" adopted mesh-store — the foundation's store is now the " + module + " module") + say(" adopted " + store.Name + " — the foundation's store is now the " + module + " module") return nil } @@ -91,7 +93,8 @@ func InstallStore(ctx context.Context, o Options, control controlPlane, // be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker // and the vault each need it or need the shape, and three copies of it drifted. func installProvider(ctx context.Context, o Options, control controlPlane, module string, - manifest []byte, buildNote string, beforePush func() error, say func(string)) error { + manifest []byte, buildNote string, beforePush func() error, say func(string), + send ...func() error) error { remote := "/" + module + "-module.json" if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { @@ -130,6 +133,11 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul return err } } + // A module whose first send is not a plain push says how it is sent: the bus, whose replacement is a + // planned step (ADR 0236), is sent by that step (adoptTheBus). + if len(send) > 0 && send[0] != nil { + return send[0]() + } _, err := pushNode(ctx, o, control, say) return err } @@ -165,13 +173,22 @@ const BusModule = "nats" // (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the // bridge to here and is not run again. // -// **What is compared is the container's name, not its image.** The store's module pins the upstream -// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's -// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is -// not the foundation's container by name would raise a second bus beside the first, and is refused -// by name before anything is registered. +// **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container +// name, the same streams directory, configuration directory, user list and certificate. That is checked +// here against the module's own manifest before anything is registered, and a difference is refused +// naming each one — under another name or on other paths the module would raise a second bus beside the +// first, or an empty one in its place. +// +// **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream +// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot +// tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on +// the module's image, on the same streams, users and certificate. The mesh replaces its bus only as a +// planned step (ADR 0236), so the module is sent by that step: its streams snapshotted first, the restart +// said, and the installer waits until the container runs the module's image, the bus answers, and the +// controller ends the step done (bus_adoption.go), so nothing after this step dials a bus that is restarting. func InstallBroker(ctx context.Context, o Options, control controlPlane, - foundation *declaration.Declaration, say func(string)) error { + foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error), + say func(string)) error { const module = BusModule manifest, err := readManifest(o.Catalogue, module) @@ -182,12 +199,18 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, if err != nil { return err } - if err := busServerIsTheFoundations(manifest, bus, module); err != nil { + if err := busLayoutIsTheModules(manifest, foundation, module); err != nil { return err } say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept") if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)", - nil, say); err != nil { + nil, say, func() error { return adoptTheBus(ctx, o, control, bus, say) }); err != nil { + return err + } + if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil { + return err + } + if err := waitForTheBusStep(ctx, o, control, say); err != nil { return err } say(" adopted " + bus.Name + " — the foundation's bus is now the " + module + @@ -195,41 +218,190 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane, return nil } -// busServerIsTheFoundations checks that the bus module's server container is the one the -// foundation raised, by name, so the applier takes it over rather than raising a second bus. -func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error { +// busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own +// loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only +// once the server is up and JetStream has its streams back. +const busMonitor = "http://127.0.0.1:8222/healthz" + +// runningImage is the image the named container was created from, or "" when there is none. +func runningImage(ctx context.Context, control controlPlane, name string) string { + asking, cancel := context.WithTimeout(ctx, control.timeout) + defer cancel() + out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name) + if err != nil { + return "" + } + return strings.TrimSpace(out) +} + +// waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised +// it from — the module's, after the applier's one recreate — and the bus answers on its monitor. +// +// Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is +// refused, naming what it runs: the push was sent and the node has not applied it, and every step after +// this one reaches the mesh through that bus. +func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container, + fetch func(context.Context, string) (int, string, error), say func(string)) error { + + deadline := time.Now().Add(o.Wait) + var running, said string + for { + running = runningImage(ctx, control, bus.Name) + if running != "" && running != bus.Image { + code, body, err := fetch(ctx, busMonitor) + if err == nil && code == http.StatusOK { + say(" bus answers " + bus.Name + " on " + running) + return nil + } + said = strings.TrimSpace(body) + if err != nil { + said = err.Error() + } + } + if !time.Now().Before(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(answerEvery): + } + } + switch { + case running == "": + return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+ + "after this step could reach the mesh. The node-engine's journal says why it did not raise it", + bus.Name, BusModule) + case running == bus.Image: + return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+ + "has not applied it. Its streams are untouched; run this installer again once the node-engine "+ + "applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule) + } + return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s", + bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said)) +} + +// BusUsersID is the bundle's resource for the bus's first user list. +const BusUsersID = "bus-accounts" + +// busLayout is where a bus lives: its container, the machine's directories it mounts as its streams, +// its configuration and its certificate, and the file its user list is. +type busLayout struct { + Container, Streams, Conf, Certificate, Users string +} + +// Where the server reads each, inside its container: the same in the bundle and the module, since both +// run the same server with the same configuration. +const ( + busStreamsInside = "/data" + busConfInside = "/etc/nats" + busCertificateInside = "/tls" +) + +// foundationBusLayout reads the layout of the bus the bundle raises. +func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) { + bus, err := brokerIn(foundation) + if err != nil { + return busLayout{}, err + } + at := mountedFrom(bus.Volumes) + l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside], + Certificate: at[busCertificateInside]} + for _, r := range foundation.Resources { + if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID { + l.Users = f.Path + } + } + return l, nil +} + +// moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of +// that container with every `${access:}` resolved to the access's path, and its `bus-users`. +func moduleBusLayout(manifest []byte, module string) (busLayout, error) { var m struct { + BusUsers string `json:"bus-users"` Resources []struct { - Type string `json:"type"` - Name string `json:"name"` + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Volumes []string `json:"volumes"` } `json:"resources"` + Accesses []struct { + ID string `json:"id"` + Path string `json:"path"` + } `json:"accesses"` } if err := json.Unmarshal(manifest, &m); err != nil { - return fmt.Errorf("the %s module's manifest is not readable: %w", module, err) + return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err) } - var names []string - for _, r := range m.Resources { - if r.Type != "container" { - continue + var containers []int + for i, r := range m.Resources { + if r.Type == "container" { + containers = append(containers, i) } - if r.Name == bus.Name { - return nil + } + if len(containers) != 1 { + var names []string + for _, i := range containers { + names = append(names, m.Resources[i].Name) } - names = append(names, r.Name) + return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+ + "declares one container", module, describeNames(names)) + } + server := m.Resources[containers[0]] + resolved := make([]string, 0, len(server.Volumes)) + for _, v := range server.Volumes { + for _, a := range m.Accesses { + v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path) + } + resolved = append(resolved, v) + } + at := mountedFrom(resolved) + return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside], + Certificate: at[busCertificateInside], Users: m.BusUsers}, nil +} + +// busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by +// field, so the applier takes it over rather than raising a second bus beside it or an empty one in its +// place (novox/hq ADR 0333 §1). Every difference is named, with both sides. +func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error { + theirs, err := moduleBusLayout(manifest, module) + if err != nil { + return err + } + ours, err := foundationBusLayout(foundation) + if err != nil { + return err + } + var differ []string + for _, f := range []struct{ what, module, bundle string }{ + {"the server container", theirs.Container, ours.Container}, + {"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams}, + {"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf}, + {"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate}, + {"its user list", theirs.Users, ours.Users}, + } { + if f.module == "" || f.module != f.bundle { + differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what, + f.module, f.bundle)) + } + } + if len(differ) == 0 { + return nil } return fmt.Errorf( - "the %s module's server is %s, and the foundation's bus is the container %q.\n"+ - "The module adopts the foundation's bus in place, so its server has to carry the name the "+ - "foundation raised; under another name it would raise a second bus beside the first, and "+ - "the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered", - module, describeNames(names), bus.Name) + "the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+ + "The module adopts the foundation's bus in place, so the bundle raises it with the module's "+ + "name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+ + "one on other paths (novox/hq ADR 0333 §1). Nothing has been registered", + module, strings.Join(differ, "\n - ")) } func describeNames(names []string) string { if len(names) == 0 { return "no container at all" } - return "the container " + strings.Join(names, ", ") + return "the containers " + strings.Join(names, ", ") } // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to diff --git a/internal/bootstrap/phase3_test.go b/internal/bootstrap/phase3_test.go index e5de53d..d0eafc1 100644 --- a/internal/bootstrap/phase3_test.go +++ b/internal/bootstrap/phase3_test.go @@ -2,10 +2,16 @@ package bootstrap import ( "context" + "encoding/json" + "fmt" + "net/http" "os" "path/filepath" "strings" "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" ) // aCatalogue is a catalogue checkout holding these modules' manifests. @@ -24,32 +30,74 @@ func aCatalogue(t *testing.T, manifests map[string]string) string { return dir } -// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name. -func aBusModule(server string) string { +// aBusModule is the nats module's manifest as far as adoption reads it: its server container, the +// directories it mounts, its certificate through an access, and its user list — the shape of the +// catalogue's modules/nats/module.json. +func aBusModule(l busLayout) string { return `{"module": "nats", "version": "1", "claims": [{"name": "mesh-broker", "scope": "mesh"}], - "bus-users": "/var/lib/nats-module/conf/accounts.conf", - "resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}], + "bus-users": "` + l.Users + `", + "resources": [ + {"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"], + "volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}], + "accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}], "build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}` } -func busStep(t *testing.T, catalogue string) (*asked, error) { +// theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats). +var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats", + Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls", + Users: "/var/lib/nats-module/conf/accounts.conf"} + +// theModulesImage is what the bus runs once the applier recreated it on the module's built image. +const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333" + +// aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the +// module's image, and its monitor says ok. +func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) { + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n", + })} + return runtime, func(_ context.Context, url string) (int, string, error) { + if url != busMonitor { + return 0, "", fmt.Errorf("asked %s", url) + } + return http.StatusOK, `{"status":"ok"}`, nil + } +} + +func busStepWith(t *testing.T, o Options, runtime *asked, + fetch func(context.Context, string) (int, string, error)) ([]string, error) { t.Helper() + aGenesisBusOnDisk(t) rewritten, err := Rewrite(theRealBundle(t), held) if err != nil { t.Fatal(err) } - runtime := &asked{answer: aMeshThatAgrees(nil)} - control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0} - err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {}) + control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} + var said []string + err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch, + func(line string) { said = append(said, line) }) + return said, err +} + +func busStep(t *testing.T, catalogue string) (*asked, error) { + t.Helper() + runtime, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + _, err := busStepWith(t, o, runtime, fetch) return runtime, err } // The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module // the mesh goes on managing, registered from the catalogue — never lavinmq, which left it. func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) { - catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")}) - runtime, _ := busStep(t, catalogue) + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + runtime, err := busStep(t, catalogue) + if err != nil { + t.Fatal(err) + } if !runtime.ran("module add /nats-module.json") { t.Fatalf("the nats module was not registered: %v", runtime.commands) } @@ -74,15 +122,23 @@ func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) { } } -// A bus module whose server is not the foundation's container by name would raise a second bus -// beside the first; it is refused naming both, before anything is registered. -func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) { - catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")}) +// A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus +// beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every +// difference with both sides, before anything is registered. The layout is the one the bundle had +// before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590). +func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) { + before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf", + Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"} + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)}) runtime, err := busStep(t, catalogue) if err == nil { - t.Fatal("a bus module with another server was adopted") + t.Fatal("a bus module kept elsewhere was adopted") } - for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} { + for _, named := range []string{ + `"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`, + `"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`, + `"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`, + } { if !strings.Contains(err.Error(), named) { t.Errorf("the refusal does not name %s: %v", named, err) } @@ -91,3 +147,301 @@ func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) { t.Errorf("something was registered: %v", runtime.commands) } } + +// One field moved is refused as surely as all of them: a module whose streams are elsewhere would be +// raised empty on the bus's own name. +func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) { + moved := theModulesLayout + moved.Streams = "/srv/nats" + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)}) + runtime, err := busStep(t, catalogue) + if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") { + t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err) + } + if strings.Contains(err.Error(), "certificate") { + t.Errorf("the refusal names what agrees: %v", err) + } + if runtime.ran("module add") { + t.Errorf("something was registered: %v", runtime.commands) + } +} + +// The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step +// gives it the setting that maps that to the port the bundle published, 5671 unless given — before the +// push, so the first declaration already names it. +func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) { + for _, c := range []struct { + given int + want string + }{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + runtime, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Ports = FoundationPorts{Bus: c.given} + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + set, pushed := -1, -1 + for i, command := range runtime.commands { + if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 { + set = i + } + if strings.Contains(command, "push anchor") && pushed < 0 { + pushed = i + } + } + if set < 0 || pushed < 0 || set > pushed { + t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands) + } + if !strings.Contains(strings.Join(said, "\n"), "settings nats on anchor: "+c.want) { + t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n")) + } + } +} + +// The one restart is said before it happens, and the step goes on only once the bus runs the module's +// image and answers (novox/hq ADR 0333 §3). +func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image + + // The push is sent; the node takes a few passes to recreate the bus, and the server a few more to + // answer. + pushed, inspected, monitored := false, 0, 0 + runtime := &asked{} + agrees := aMeshThatAgrees(nil) + runtime.answer = func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "bus upgrade --why"): + pushed = true + case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"): + if !pushed { + return bundleImage + "\n", nil + } + inspected++ + if inspected < 3 { + return bundleImage + "\n", nil + } + return theModulesImage + "\n", nil + } + return agrees(name, args) + } + fetch := func(context.Context, string) (int, string, error) { + monitored++ + if monitored < 3 { + return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil + } + return http.StatusOK, `{"status":"ok"}`, nil + } + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Wait = 10 * time.Second + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + text := strings.Join(said, "\n") + restart := strings.Index(text, "restarting mesh-broker-nats — once") + answers := strings.Index(text, "bus answers mesh-broker-nats on "+theModulesImage) + adopted := strings.Index(text, "adopted mesh-broker-nats") + if restart < 0 || answers < restart || adopted < answers { + t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text) + } + if inspected < 3 || monitored < 3 { + t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+ + "times, its monitor %d)", inspected, monitored) + } +} + +// A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after +// this one reaches the mesh through that bus, and the push was not applied. +func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) { + previous := answerEvery + answerEvery = time.Millisecond + defer func() { answerEvery = previous }() + + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image + runtime := &asked{answer: aMeshThatAgrees(map[string]string{ + "inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n", + })} + fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil } + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + o.Wait = 20 * time.Millisecond + _, err = busStepWith(t, o, runtime, fetch) + if err == nil { + t.Fatal("a bus the node never recreated was taken as adopted") + } + if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") { + t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err) + } +} + +// A re-run after the adoption finds the bus on the module's image already, and says so rather than +// announcing a restart that does not happen. +func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) { + catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)}) + runtime, fetch := aBusThatAdopts() + o := installing(t, catalogue) + o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"} + said, err := busStepWith(t, o, runtime, fetch) + if err != nil { + t.Fatal(err) + } + text := strings.Join(said, "\n") + if strings.Contains(text, "restarting") || !strings.Contains(text, "already adopted mesh-broker-nats") { + t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text) + } +} + +// theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333, +// "How it is checked"): in a merge check the clone the build seat puts beside this repository +// (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue +// beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check +// never skips it. +func theCataloguesBusModule(t *testing.T) []byte { + t.Helper() + const inside = "mesh-catalog/modules/nats/module.json" + if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" { + raw, err := os.ReadFile(filepath.Join(root, inside)) + if err != nil { + t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+ + "cannot be held to the module's: %v", root, err) + } + t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside)) + return raw + } + beside := filepath.Join("..", "..", "..", inside) + raw, err := os.ReadFile(beside) + if err != nil { + t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+ + "bus is not held to the nats module's manifest here; the merge check holds it", beside) + } + t.Logf("judged against the catalogue checked out at %s", beside) + return raw +} + +// **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's +// manifest: the bus's container, its streams, its configuration, its certificate and its user list are +// where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised +// `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590). +func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) { + manifest := theCataloguesBusModule(t) + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil { + t.Fatal(err) + } + // And the port the installer's setting maps is the one the module declares (ADR 0333 §2). + var m struct { + Resources []struct { + Type string `json:"type"` + Ports []string `json:"ports"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + declared := false + for _, r := range m.Resources { + for _, p := range r.Ports { + if p == fmt.Sprintf("%d:4222", busModulePort) { + declared = true + } + } + } + if !declared { + t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+ + "to the node's bus port", busModulePort) + } +} + +// The bundle as the installer reads it agrees with the module's layout as this repository states it, +// and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the +// bus reads, and makes the container that reads it re-read it. +func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + ours, err := foundationBusLayout(rewritten.Declaration) + if err != nil { + t.Fatal(err) + } + if ours != theModulesLayout { + t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout) + } + if ours.Users != busAccounts || ours.Container != busContainer { + t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s", + busAccounts, busContainer, ours.Users, ours.Container) + } + if filepath.Dir(ours.Users) != ours.Conf { + t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf) + } +} + +// The bundle's bus configuration and user list are seeds (create-once): an installer run again after +// the adoption applies the bundle again, and must not put the bundle's one user back over the users the +// module writes there (ADR 0333 §4). +func TestTheBundlesBusFilesAreSeeds(t *testing.T) { + rewritten, err := Rewrite(theRealBundle(t), held) + if err != nil { + t.Fatal(err) + } + seeds := map[string]bool{} + for _, r := range rewritten.Declaration.Resources { + if f, ok := r.(*declaration.File); ok && f.CreateOnce { + seeds[f.Path] = true + } + } + for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} { + if !seeds[path] { + t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path) + } + } +} + +// aGenesisBusOnDisk puts what the adoption reads on the machine in a test's own directory: the bus's +// certificate, the controller's bus credential, and where the snapshot goes. The step's wait is short. +func aGenesisBusOnDisk(t *testing.T) { + t.Helper() + dir := t.TempDir() + cert := filepath.Join(dir, "tls.crt") + if err := os.WriteFile(cert, []byte(aCertificatePEM), 0o644); err != nil { + t.Fatal(err) + } + secret := filepath.Join(dir, "bus-controller.secret") + if err := os.WriteFile(secret, []byte("BUS-PW"), 0o600); err != nil { + t.Fatal(err) + } + was := []string{busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret} + wasWithin := busStepWithin + busSnapshotFile, busSnapshotCredential = filepath.Join(dir, "bus.tar"), filepath.Join(dir, "bus.credential") + busCertificateFile, busControllerSecret = cert, secret + busStepWithin = 50 * time.Millisecond + t.Cleanup(func() { + busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret = was[0], was[1], was[2], was[3] + busStepWithin = wasWithin + }) +} + +// aCertificatePEM is a PEM block standing for the bus's certificate; its pin is the sha256 of these bytes. +const aCertificatePEM = "-----BEGIN CERTIFICATE-----\nAAECAwQFBgcICQ==\n-----END CERTIFICATE-----\n" diff --git a/internal/bootstrap/phase_packages.go b/internal/bootstrap/phase_packages.go index cb3541d..e60036b 100644 --- a/internal/bootstrap/phase_packages.go +++ b/internal/bootstrap/phase_packages.go @@ -2,6 +2,7 @@ package bootstrap import ( "context" + "encoding/json" "fmt" "net/http" "os" @@ -24,7 +25,7 @@ import ( const ( // foundationStore is the foundation's postgres container — the mesh's own memory, raised from the // bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051). - foundationStore = "mesh-store" + foundationStore = "postgres" // giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module — // under the name the gitea MODULE declares for its container, so the module finds it and holds // it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090). @@ -49,8 +50,8 @@ const ( packagesTeam = "packages" // giteaAdminUser is the admin the bootstrap creates and the provisioner later authenticates as. giteaAdminUser = "mesh-admin" - // builderGiteaUser is the gitea account the builder publishes and pulls with at genesis. It is - // the `as` the builder's static package binding names. + // builderGiteaUser is the gitea account the build agent publishes and pulls with at genesis: the + // `as` of the genesis document (novox/hq ADR 0336 §3). Retiring it is issue 602. builderGiteaUser = "mesh-builder" // giteaDBRole/giteaDBName is gitea's own database in the foundation store. giteaDBRole = "mesh_gitea" @@ -133,8 +134,8 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro return err } - say(" delivering the builder its registry credential") - if err := deliverBuilderNpm(ctx, o, control, builderPassword, say); err != nil { + say(" sealing the build agent the genesis document for the forge it raised") + if err := deliverGenesisRegistry(ctx, o, control, genesisRegistry(ports, builderPassword), say); err != nil { return err } @@ -362,27 +363,69 @@ func packagePasswords() (db, admin, builder string, err error) { return db, admin, builder, nil } -// deliverBuilderNpm seals the builder's registry password to this node as its `npm-password` -// own-secret, the same way the control plane's store connections are delivered — carry the value in, -// `secret accept`, and the next push writes it sealed where the builder reads it. -func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string, - say func(string)) error { - at := "/accepting-npm-password" - if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { - return err - } - if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil { - return err - } - // Push so the sealed secret reaches the builder, which restarts on it and comes back credentialed. - if _, err := control.tell(ctx, "push", o.Node); err != nil { - return err - } - return nil +// GenesisRegistrySecret is the build agent's own-secret the genesis document is sealed as (novox/hq +// ADR 0336 §3). The build-agent manifest declares it, issued outside the mesh: only this installer +// gives one, on genesis's node. +const GenesisRegistrySecret = "genesis-package-registry" + +// GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): the forge this step raised, the port +// this node gave the package registry, its npm path, and the account this step made with its password. +// mesh-controller's cmd/mesh-builder reads it under the same field names while the build agent's +// npm-package-registry requirement is unbound, and prefers the binding once the gitea module is +// assigned and it binds. +type GenesisRegistry struct { + Scheme string `json:"scheme"` + At string `json:"at"` + Port int `json:"port"` + NpmPath string `json:"npm-path"` + As string `json:"as"` + Password string `json:"password"` } -// PublishTheSDK dispatches a build of the SDK to the builder. The builder has its registry -// credential by now, so the build's `npm publish` authenticates; the artifact is a `package`, built +// genesisRegistry is the document for the forge raised here: on the machine's own loopback, where it +// answers on the host network and where its ROOT_URL says it is (raiseGiteaServer), on the port this +// node gave the package registry. +func genesisRegistry(ports FoundationPorts, password string) GenesisRegistry { + return GenesisRegistry{Scheme: "http", At: "127.0.0.1", Port: ports.Packages, + NpmPath: "/api/packages/" + packagesOrg + "/npm/", As: builderGiteaUser, Password: password} +} + +// deliverGenesisRegistry seals the genesis document to the build agent on this node as its +// `genesis-package-registry` own-secret (novox/hq ADR 0336 §3), the same way the control plane's store +// connections are delivered — carry the value in, `secret accept`, and the next push writes it sealed +// where the build agent reads it. +// +// **The build agent's one way to the forge until the registry binds.** Its npm-package-registry +// requirement is optional (ADR 0336 §1), and nothing provides it until the gitea module is assigned +// after the base, so the sdk step (15) publishes and the base step (16) resolves with the account in +// this document. It replaces the retired builder's `npm-password` own-secret and the `serves.port` +// setting its carried binding took, neither of which the build agent declares. +func deliverGenesisRegistry(ctx context.Context, o Options, control controlPlane, doc GenesisRegistry, + say func(string)) error { + raw, err := json.Marshal(doc) + if err != nil { + return err + } + at := "/accepting-" + GenesisRegistrySecret + if err := control.carryingSecret(ctx, "mesh-accepting-"+GenesisRegistrySecret, raw, at); err != nil { + return err + } + if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, GenesisRegistrySecret, "--from", at); err != nil { + return err + } + // Push so the sealed document reaches the build agent, which reads it at its next build — and, + // like every push before the adoption, the bus hears the users it carries (issue 608). + // + // **Waited for until the node applied it** (issue 615). The next step asks the build agent to + // publish the SDK, and the agent reads the document from the file the node writes when it applies + // this push. A push that returned on sending let the ask race the apply: one genesis found the + // document in time, the next was refused, "neither its binding nor the genesis document". + _, err = pushNodeApplied(ctx, o, control, say) + return err +} + +// PublishTheSDK dispatches a build of the SDK to the build agent. The agent has the genesis document by +// now (novox/hq ADR 0336 §7), so the build's `npm publish` authenticates; the artifact is a `package`, built // on a public base, so this needs no toolchain — which is the whole point of doing it before the base. func PublishTheSDK(ctx context.Context, o Options, control controlPlane, say func(string)) error { if o.SDKSource.Repository == "" { diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go index efbe5e9..dbedf50 100644 --- a/internal/bootstrap/ports.go +++ b/internal/bootstrap/ports.go @@ -117,24 +117,6 @@ const busModulePort = 4222 // PortsSetting is the controller's settings key for a module's given ports. const PortsSetting = "ports" -// ServesSetting is the settings key that carries what a provider serves, inside the one binding a -// module carries rather than resolves — the builder's package binding (novox/hq 04-ISSUES/085). -// -// **A port given at genesis has to be a setting wherever it is read, or something puts it back.** -// The builder reaches the package registry through a binding written in its own manifest, because -// at genesis no module yet provides `package-registry` and so there is nothing for the controller -// to resolve it from. That binding used to be rewritten, as text, when the installer registered the -// builder — which a later registration from the catalogue undid silently. Set as a node's setting -// instead, it is held by the controller rather than by the manifest, so re-registering the builder -// leaves it where it was. -// -// **A settings layer is the module's, not one resource's.** The controller lays it over every file -// of that module which merges as JSON, so `serves` lands in the package binding only because that -// binding is the builder's one mergeable file. A second mergeable file added to the builder would -// be given a `serves` key too, meaning nothing to whatever reads it. Worth knowing before adding -// one. -const ServesSetting = "serves" - // setFoundationSettings tells the controller the ports this node gave a foundation module — and, // on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it // has a private-network address (novox/hq ADR 0100). Done after the module is registered and before @@ -148,13 +130,8 @@ func setFoundationSettings(ctx context.Context, o Options, control controlPlane, if o.Adopted && module == RegistryModule { values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"} } - if packages := o.Ports.orDefaults().Packages; module == BuilderModule && - packages != DefaultPorts().Packages { - // The one binding nothing resolves: the builder dials the forge by a number it carries. - // The rest of the binding — the provision, the forge it is from, the account it presents — - // is the manifest's and stays there. - values[ServesSetting] = map[string]int{"port": packages} - } + // No `serves` setting for the build agent: the port given for the package registry reaches it in + // the genesis document (novox/hq ADR 0336 §3), and from the binding once the registry binds. if len(values) == 0 { return nil } diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index fed87dd..661fdce 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -68,11 +68,11 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) { if got.Places == 0 { t.Fatal("nothing was rewritten") } - storeC := containerNamed(r.Declaration, "mesh-store") + storeC := containerNamed(r.Declaration, "postgres") if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { t.Errorf("the store publishes %v", storeC.Ports) } - bus := containerNamed(r.Declaration, "mesh-broker") + bus := containerNamed(r.Declaration, "mesh-broker-nats") if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" { t.Errorf("the bus publishes %v", bus.Ports) } @@ -239,6 +239,12 @@ func (c *controlRecorder) run(_ context.Context, name string, args ...string) (s } if name == "docker" && args[0] == "exec" { c.told = append(c.told, strings.Join(args[3:], " ")) + if strings.Join(args[3:], " ") == "broker accounts" { + return "accounts {}\n", nil // the composed user list, placed after every push (issue 608) + } + } + if name == "sh" { + c.told = append(c.told, "placed the bus's users") } return "", nil } diff --git a/internal/bootstrap/push_places_users_test.go b/internal/bootstrap/push_places_users_test.go new file mode 100644 index 0000000..d42912a --- /dev/null +++ b/internal/bootstrap/push_places_users_test.go @@ -0,0 +1,134 @@ +package bootstrap + +import ( + "context" + "strings" + "testing" + "time" +) + +// Defends novox/hq issue 608 (ADR 0331 §5, ADR 0333 §4): until the bus is adopted as the nats module, +// every push genesis makes is followed by the installer placing the composed user list where the bus +// reads it, and making the bus re-read it. The build agent's account was minted at step 13 and sealed +// with that push, and the list was placed only at enrolment, so the bus refused the agent +// ("Authorization Violation") and genesis stopped at step 15. + +// placing answers the controller as a mesh whose bus is or is not the nats module yet, and records +// what was run, in order. +func placing(adopted bool) (*asked, controlPlane) { + listed := "build-agent 1\n" + if adopted { + listed += "nats 1\n" + } + runtime := &asked{} + runtime.answer = func(name string, args []string) (string, error) { + joined := strings.Join(args, " ") + switch { + case name == "sh": + return "", nil + case strings.HasSuffix(joined, "module list"): + return listed, nil + case strings.HasSuffix(joined, "broker accounts"): + return "accounts {\n MESH { users = [ { user: \"anchor.build-agent\" } ] }\n}\n", nil + } + return "", nil + } + return runtime, controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second} +} + +// placedAfterThePush is whether the bus's users were placed — written at the module's bus-users path, +// the bus told to re-read them — after the push, and how often. +func placedAfterThePush(t *testing.T, commands []string) int { + t.Helper() + pushed, placed := -1, 0 + for i, c := range commands { + if strings.Contains(c, "/mesh-controller push anchor") && pushed < 0 { + pushed = i + } + if strings.HasPrefix(c, "sh -c") && strings.Contains(c, busAccounts) && + strings.Contains(c, "kill -s HUP "+busContainer) { + if pushed < 0 || i < pushed { + t.Fatalf("the bus's users were placed before the push that carries the account: %v", commands) + } + placed++ + } + } + if pushed < 0 { + t.Fatalf("nothing was pushed: %v", commands) + } + return placed +} + +func TestTheBusHearsTheBuildAgentsAccountAfterItsPush(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(false) + o := Options{Node: "anchor", Wait: time.Second} + if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if n := placedAfterThePush(t, runtime.commands); n != 1 { + t.Fatalf("the bus's users were placed %d time(s) after the build agent's push; once: %v", n, runtime.commands) + } +} + +func TestTheGenesisDocumentsPushIsHeardByTheBusToo(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(false) + o := Options{Node: "anchor", Wait: time.Second} + if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil { + t.Fatal(err) + } + if n := placedAfterThePush(t, runtime.commands); n != 1 { + t.Fatalf("the bus's users were placed %d time(s) after the genesis document's push; once: %v", n, runtime.commands) + } +} + +// From the adoption on the nats module writes the list, and the installer does not (ADR 0333 §4). +func TestOnceTheBusIsTheNatsModulesThePushPlacesNothing(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(true) + o := Options{Node: "anchor", Wait: time.Second} + if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + if n := placedAfterThePush(t, runtime.commands); n != 0 { + t.Fatalf("the installer placed the bus's users over the nats module's: %v", runtime.commands) + } +} + +// Defends novox/hq issue 615: the packages step's push is waited for until the node applied it, because +// the sdk step right after asks the build agent for a build that reads the genesis document from the +// file that apply writes. A push that returned on sending let the ask race the apply. +func TestThePackagesStepWaitsUntilTheNodeAppliedTheGenesisDocument(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(false) + o := Options{Node: "anchor", Wait: 3 * time.Minute} + if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil { + t.Fatal(err) + } + for _, c := range runtime.commands { + if strings.Contains(c, "/mesh-controller push anchor") { + if !strings.HasSuffix(c, "push anchor --wait 3m0s") { + t.Fatalf("the genesis document's push does not wait for the node to apply it: %s", c) + } + return + } + } + t.Fatalf("the genesis document was never pushed: %v", runtime.commands) +} + +// Every other push of genesis still returns on sending: only a push whose content the next step reads on +// the machine waits. +func TestAnOrdinaryPushDoesNotWait(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + runtime, control := placing(false) + o := Options{Node: "anchor", Wait: time.Second} + if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil { + t.Fatal(err) + } + for _, c := range runtime.commands { + if strings.Contains(c, "push anchor --wait") { + t.Fatalf("an ordinary push waits: %s", c) + } + } +} diff --git a/internal/bootstrap/registry_test.go b/internal/bootstrap/registry_test.go index d05bb83..0fbc676 100644 --- a/internal/bootstrap/registry_test.go +++ b/internal/bootstrap/registry_test.go @@ -58,8 +58,20 @@ func aMeshThatAgrees(answers map[string]string) func(string, []string) (string, } } switch { + case name == "sh" && len(args) == 2 && strings.Contains(args[1], "accounts.conf"): + return "", nil // the bus's users placed after a push (issue 608) + case name == "sh" && len(args) == 2 && strings.Contains(args[1], "mesh-nats-snapshot"): + return "", nil // the streams snapshotted before the bus's planned step (issue 590) + case name == "docker" && strings.HasSuffix(joined, "/mesh-controller bus"): + return "the bus is nats, built 53e51864, on anchor\nlast step 1: not known → 53e51864 on anchor by installer (" + + adoptionWhy + "): done at 2026-10-13 11:40; snapshot " + busSnapshotFile + "\n", nil + case name == "docker" && strings.Contains(joined, "/mesh-controller builds nats -n 1"): + return "nats built 53e51864 anchor\n image artifact-store://nats/server@sha256:" + + strings.Repeat("4", 64) + "\n bundle artifact-store://nats/tools/blobs/sha256:" + strings.Repeat("5", 64) + "\n", nil case name != "docker": return "", fmt.Errorf("unexpected program %q", name) + case strings.HasSuffix(joined, "broker accounts"): + return "accounts {}\n", nil case args[0] == "cp": return "", nil case args[0] == "container": diff --git a/internal/bootstrap/rootsecrets.go b/internal/bootstrap/rootsecrets.go index 9e63874..9f1fa1e 100644 --- a/internal/bootstrap/rootsecrets.go +++ b/internal/bootstrap/rootsecrets.go @@ -47,6 +47,9 @@ const ( // StoreSuperuserFile is where the store's superuser password lives on the machine — the // postgres module's own-secret path, so genesis and adoption write the same file. StoreSuperuserFile = "/var/lib/postgres/superuser.secret" + // StoreDataDir is where the store keeps its data — the postgres module's data directory, so + // genesis raises the store where the module adopts it (novox/hq ADR 0338 §1). + StoreDataDir = "/var/lib/mesh-store" // BusControllerFile is where the controller's password on the bus lives on the machine. Not a // module's path: no module reads it — the permanent controller is handed the address the // temporary one was given — and genesis keeps it only so a second run puts the same account @@ -62,7 +65,7 @@ const ( // What the template says, matched exactly. A template that says something else is a template // this installer does not know how to make safe, and it says so rather than guessing. templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` - templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]` + templateStoreVolumes = `"volumes": ["` + StoreDataDir + `:/var/lib/postgresql/data"]` templateStoreURL = "postgres:bootstrap@" // BusURLPlaceholder is the controller's account in its bus address, as the template names it. @@ -265,8 +268,8 @@ func Masking(say func(string), c RootCredentials) func(string) { // RefuseExistingServers stops a run that would put a made credential in front of a store raised // by an earlier installer with the template's. // -// The store's password is set by initdb, once, on an empty volume. A machine that already holds -// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new +// The store's password is set by initdb, once, on an empty data directory. A machine that already holds +// a cluster in the store's data directory and has no credential file was raised with `bootstrap`, and minting a new // value here would make a bundle that dials with a password the server does not have — failing // three steps later, in the schemas, with nothing pointing back here. Refused by name instead, // with the way forward. @@ -278,11 +281,14 @@ func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) e if !c.StoreMade { return nil } - if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil { - return nil // no such volume: a fresh machine, which is the case this installer makes + // Where the store keeps its data, as the postgres module does (novox/hq ADR 0338 §2): a cluster + // there is one initdb already set a password for. The docker volume genesis used before is not + // the store's data any more, and a machine holding only that is not refused. + if _, err := run(ctx, "test", "-e", StoreDataDir+"/pgdata/PG_VERSION"); err != nil { + return nil // no cluster there: a fresh machine, which is the case this installer makes } return fmt.Errorf( - "this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+ + "this machine already holds the store's data (in "+StoreDataDir+") and no credential at %s, so it was raised "+ "by an earlier installer with the template's password. A new one made here would not open it. "+ "Put the password the store has into %s (0600, the value alone) and run again; then change it "+ "on the server and accept the new value — this installer does not rotate a running store", @@ -311,7 +317,7 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) { return out, err } if bundle, err = replaceOnce(bundle, templateStoreVolumes, - `"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, + `"volumes": ["`+StoreDataDir+`:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, "the store's volumes"); err != nil { return out, err } diff --git a/internal/bootstrap/rootsecrets_test.go b/internal/bootstrap/rootsecrets_test.go index c0db52a..29c61e1 100644 --- a/internal/bootstrap/rootsecrets_test.go +++ b/internal/bootstrap/rootsecrets_test.go @@ -59,7 +59,7 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) { switch x := res.(type) { case *declaration.Container: switch x.Name { - case "mesh-store": + case "postgres": store = true if _, has := x.Env["POSTGRES_PASSWORD"]; has { t.Error("the store still takes its password from its environment") diff --git a/internal/bootstrap/store_layout_test.go b/internal/bootstrap/store_layout_test.go new file mode 100644 index 0000000..809ded7 --- /dev/null +++ b/internal/bootstrap/store_layout_test.go @@ -0,0 +1,163 @@ +package bootstrap + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0338 (issue 609): genesis raises its store as the postgres module's server, so +// the store step adopts it as it runs. Until then the bundle raised `mesh-store`, on another image, in a +// docker volume, and genesis stopped at step 17 of 24. + +// theStoreGenesisRaises is the store container of the bundle as the installer applies it: rewritten for +// this machine's credentials, so its password is the file the module reads too. +func theStoreGenesisRaises(t *testing.T) *declaration.Container { + t.Helper() + r := producedBundle(t) + store, err := storeIn(r.Declaration) + if err != nil { + t.Fatal(err) + } + return store +} + +// The bundle's store is exactly the catalogue's postgres module's server, read from the module's own +// manifest (ADR 0338 §1): its name, its image (so a drift of either pin fails here), where its data is, +// and the file its superuser password is read from, with the environment that says so. +func TestTheBundleRaisesTheStoreAsTheCataloguesPostgresModuleDeclaresIt(t *testing.T) { + manifest := theCataloguesModule(t, "postgres") + var m struct { + Resources []struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Image string `json:"image"` + Env map[string]string `json:"env"` + Ports []string `json:"ports"` + Volumes []string `json:"volumes"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + t.Fatal(err) + } + ours := theStoreGenesisRaises(t) + found := false + for _, r := range m.Resources { + if r.Type != "container" || r.ID != "server" { + continue + } + found = true + if ours.Name != r.Name { + t.Errorf("the bundle raises the store as %q; the module's server is %q", ours.Name, r.Name) + } + if ours.Image != r.Image { + t.Errorf("the bundle raises the store on %s; the module pins %s — the two pins move together", ours.Image, r.Image) + } + for _, key := range []string{"POSTGRES_PASSWORD_FILE", "PGDATA"} { + if ours.Env[key] != r.Env[key] { + t.Errorf("the bundle's store has %s=%q; the module's %q", key, ours.Env[key], r.Env[key]) + } + } + if _, has := ours.Env["POSTGRES_PASSWORD"]; has { + t.Error("the bundle's store takes its password from its environment; the module reads a file") + } + // The module's own directory for its state is /, where genesis keeps the + // superuser's file (StoreSuperuserFile). + var theirs []string + for _, v := range r.Volumes { + theirs = append(theirs, strings.ReplaceAll(v, "${dir:state}", "/var/lib/postgres")) + } + if strings.Join(ours.Volumes, " ") != strings.Join(theirs, " ") { + t.Errorf("the bundle's store mounts %v; the module's %v", ours.Volumes, theirs) + } + if strings.Join(ours.Ports, " ") != strings.Join(r.Ports, " ") { + t.Errorf("the bundle publishes the store on %v; the module on %v", ours.Ports, r.Ports) + } + // And the store step's own check agrees: by name and image, before anything is registered. + if err := serverMatchesFoundation(manifest, ours, "postgres"); err != nil { + t.Errorf("the store step would refuse the store the bundle raises: %v", err) + } + } + if !found { + t.Fatal("the postgres module declares no server container") + } +} + +// The store's data directory is made by the bundle before the store starts, owned by the image's +// postgres user, as the module declares it (ADR 0338 §1, and the measurement under its §5). +func TestTheBundleMakesTheStoresDataDirectoryAsTheModuleDoes(t *testing.T) { + r := producedBundle(t) + at, store := -1, -1 + for i, res := range r.Declaration.Resources { + if d, ok := res.(*declaration.Directory); ok && d.Path == StoreDataDir { + at = i + if d.Mode != "0700" || d.Owner != "999:70" { + t.Errorf("the store's data directory is made %s %s; the module makes it 0700 999:70", d.Mode, d.Owner) + } + } + if res.Identity() == StoreID { + store = i + } + } + if at < 0 || store < 0 || at > store { + t.Fatalf("the store's data directory %s is not made before the store (directory at %d, store at %d)", StoreDataDir, at, store) + } +} + +// No step of the bundle and no step of the installer names `mesh-store`, which is the seat's name only +// (ADR 0338 §2): every step that reaches the store reaches the container the bundle raised. +func TestNoStepReachesTheStoreByTheSeatsName(t *testing.T) { + text := string(theRealBundle(t)) + for _, by := range []string{`"in": "mesh-store"`, `container:mesh-store`, `exec mesh-store`, `"name": "mesh-store"`, `mesh-store-data`} { + if strings.Contains(text, by) { + t.Errorf("the bundle still reaches the store as %s", by) + } + } + store := theStoreGenesisRaises(t) + if foundationStore != store.Name { + t.Errorf("the packages step reaches the store as %q; the bundle raises it as %q", foundationStore, store.Name) + } + r := producedBundle(t) + for _, res := range r.Declaration.Resources { + if a, ok := res.(*declaration.Action); ok && a.In != "" && a.In != store.Name && strings.Contains(a.ID, "database") { + t.Errorf("the step %s reaches the store in %q", a.ID, a.In) + } + } +} + +// The installer's refusal to raise a store over data it cannot open looks where the store keeps its data +// (ADR 0338 §2): a cluster there and no credential is refused; the old docker volume alone is not. +func TestTheRefusalOverExistingDataLooksInTheStoresDirectory(t *testing.T) { + made := RootCredentials{Store: "s", Bus: "b", StoreMade: true} + machine := func(cluster, volume bool) Runner { + return func(_ context.Context, name string, args ...string) (string, error) { + joined := name + " " + strings.Join(args, " ") + switch { + case joined == "test -e "+StoreDataDir+"/pgdata/PG_VERSION": + if cluster { + return "", nil + } + case strings.HasPrefix(joined, "docker volume inspect"): + if volume { + return "[]", nil + } + } + return "", errors.New("exit status 1") + } + } + err := RefuseExistingServers(context.Background(), machine(true, false), made) + if err == nil || !strings.Contains(err.Error(), StoreDataDir) { + t.Fatalf("a cluster in %s and no credential was not refused naming it: %v", StoreDataDir, err) + } + if err := RefuseExistingServers(context.Background(), machine(false, true), made); err != nil { + t.Fatalf("a machine holding only the old volume was refused: %v", err) + } + if err := RefuseExistingServers(context.Background(), machine(true, false), RootCredentials{Store: "s"}); err != nil { + t.Fatalf("a kept credential was refused: %v", err) + } +} diff --git a/internal/bootstrap/verify_test.go b/internal/bootstrap/verify_test.go index 14617b7..ddd37ab 100644 --- a/internal/bootstrap/verify_test.go +++ b/internal/bootstrap/verify_test.go @@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) { // run-once and scheduled shapes are excluded on purpose — a step that has exited is not a // fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the // plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all. - want := []string{"mesh-store", "mesh-broker", "temp-mesh-controller"} + want := []string{"postgres", "mesh-broker-nats", "temp-mesh-controller"} if len(verified.Running) != len(want) { t.Fatalf("confirmed %v running, want %v", verified.Running, want) } @@ -132,7 +132,7 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) { // than being told only that something is not what it should be. func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { runtime := &asked{answer: func(_ string, args []string) (string, error) { - if args[0] == "container" && args[len(args)-1] == "mesh-broker" { + if args[0] == "container" && args[len(args)-1] == "mesh-broker-nats" { return "false exited\n", nil } if args[0] == "container" { @@ -146,7 +146,7 @@ func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { if err == nil { t.Fatal("a container that had exited was reported as part of a running foundation") } - if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") { + if !strings.Contains(err.Error(), "mesh-broker-nats") || !strings.Contains(err.Error(), "exited") { t.Errorf("the failure does not say which container is in what state: %v", err) } } diff --git a/internal/outward/networks.go b/internal/outward/networks.go new file mode 100644 index 0000000..8075726 --- /dev/null +++ b/internal/outward/networks.go @@ -0,0 +1,92 @@ +package outward + +import ( + "errors" + "fmt" + "net" + "net/netip" + "sort" +) + +// Network is one network this machine is directly attached to, and the link it is on (novox/hq ADR +// 0340 §7): the address and prefix of the network, not of the machine's own address on it. +// +// **Every one is said, and the machine judges none of them.** Which count as the machine's local +// network — private ranges only, never a public subnet, never a link-local one — is the controller's +// judgement (ADR 0340 §2), made where the filter is composed and shown with why; a machine that left a +// network out would make that judgement silently. +type Network struct { + Link string `json:"link"` + Network string `json:"network"` +} + +// AddressesOf is the addresses on one link, as address and prefix. A parameter so a test can hold a +// machine's links without one. +type AddressesOf func(link string) ([]netip.Prefix, error) + +// RealAddresses reads one link's addresses from the kernel. +func RealAddresses(link string) ([]netip.Prefix, error) { + iface, err := net.InterfaceByName(link) + if err != nil { + return nil, err + } + addrs, err := iface.Addrs() + if err != nil { + return nil, err + } + var out []netip.Prefix + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + addr, ok := netip.AddrFromSlice(ipnet.IP) + if !ok { + continue + } + bits, _ := ipnet.Mask.Size() + out = append(out, netip.PrefixFrom(addr.Unmap(), bits)) + } + return out, nil +} + +// Networks is the networks on each of these links — the links that face outside (Links) — sorted by link +// and network, without repeats. A link that has gone since Links read it has no networks rather than +// failing the rest; a link whose addresses cannot be read fails, so the machine says the fact is unknown +// rather than reporting a partial account as the whole. +func Networks(links []string, addressesOf AddressesOf) ([]Network, error) { + if addressesOf == nil { + addressesOf = RealAddresses + } + seen := map[Network]bool{} + var out []Network + for _, link := range links { + prefixes, err := addressesOf(link) + if err != nil { + if isNoSuchInterface(err) { + continue + } + return nil, fmt.Errorf("cannot read the addresses on %s: %w", link, err) + } + for _, p := range prefixes { + n := Network{Link: link, Network: p.Masked().String()} + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Link != out[j].Link { + return out[i].Link < out[j].Link + } + return out[i].Network < out[j].Network + }) + return out, nil +} + +// isNoSuchInterface is the error net.InterfaceByName gives for a link that is not there. +func isNoSuchInterface(err error) bool { + var op *net.OpError + return errors.As(err, &op) && op.Err != nil && op.Err.Error() == "no such network interface" +} diff --git a/internal/outward/networks_test.go b/internal/outward/networks_test.go new file mode 100644 index 0000000..182d3a3 --- /dev/null +++ b/internal/outward/networks_test.go @@ -0,0 +1,59 @@ +package outward + +import ( + "errors" + "net/netip" + "reflect" + "testing" +) + +func prefixes(s ...string) []netip.Prefix { + var out []netip.Prefix + for _, p := range s { + out = append(out, netip.MustParsePrefix(p)) + } + return out +} + +// **Each link facing outside says the networks it is attached to** (novox/hq ADR 0340 §7), as the +// network's address and prefix, never the machine's own address on it — and every one, public and +// link-local included, because which count is the controller's judgement and said with why. +func TestEachOutwardLinkSaysTheNetworksItIsOn(t *testing.T) { + addrs := map[string][]netip.Prefix{ + "eth0": prefixes("192.168.1.10/24", "fe80::1/64", "fd12:3456:789a::5/64"), + "eth1": prefixes("203.0.113.7/24"), + // Not asked: the bridge is not a link facing outside. + "docker0": prefixes("172.17.0.1/16"), + } + got, err := Networks([]string{"eth1", "eth0"}, func(link string) ([]netip.Prefix, error) { return addrs[link], nil }) + if err != nil { + t.Fatal(err) + } + want := []Network{ + {Link: "eth0", Network: "192.168.1.0/24"}, + {Link: "eth0", Network: "fd12:3456:789a::/64"}, + {Link: "eth0", Network: "fe80::/64"}, + {Link: "eth1", Network: "203.0.113.0/24"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %+v\nwant %+v", got, want) + } +} + +// A link whose addresses cannot be read fails the whole, so the machine says the fact is unknown rather +// than reporting part of its networks as all of them. +func TestAnUnreadableLinkIsNotAPartialAccount(t *testing.T) { + _, err := Networks([]string{"eth0"}, func(string) ([]netip.Prefix, error) { return nil, errors.New("denied") }) + if err == nil { + t.Fatal("an unreadable link was reported as having no networks") + } +} + +// A link gone between reading the routing table and reading its addresses has no networks, and the +// rest are still said. +func TestALinkGoneSinceItWasListedHasNoNetworks(t *testing.T) { + got, err := Networks([]string{"gone0"}, RealAddresses) + if err != nil || len(got) != 0 { + t.Fatalf("got %v, %v", got, err) + } +} diff --git a/internal/profile/machine.go b/internal/profile/machine.go index f67a168..3e9a807 100644 --- a/internal/profile/machine.go +++ b/internal/profile/machine.go @@ -10,6 +10,8 @@ import ( "sync" "syscall" "time" + + "github.com/novox/mesh-host/internal/outward" ) // Machine is what this machine is made of, as it says itself (novox/hq issue 160): what somebody @@ -50,6 +52,11 @@ type Machine struct { // Filesystems is each local filesystem mounted, by where, with its size and what is free. Filesystems []Filesystem `json:"filesystems,omitempty"` + // Networks is every network the machine is directly attached to on a link facing outside it, as the + // link and the network's address and prefix (novox/hq ADR 0340 §7): what a port reached from + // `local-network` is resolved from. Every one is said; which count is the controller's judgement. + Networks []outward.Network `json:"networks,omitempty"` + // Unknown names each fact that could not be determined, and why. Unknown map[string]string `json:"unknown,omitempty"` } @@ -79,11 +86,12 @@ const ( FactTimezone = "timezone" FactFilesystems = "filesystems" FactPublicAddress = "public address" + FactNetworks = "networks" ) // MachineFacts is every fact Unknown may name, in the order a reader shows them. var MachineFacts = []string{FactKernelRelease, FactDistribution, FactVersion, FactProcessor, FactCores, - FactThreads, FactMemory, FactVirtual, FactUptime, FactTimezone, FactFilesystems, FactPublicAddress} + FactThreads, FactMemory, FactVirtual, FactUptime, FactTimezone, FactFilesystems, FactNetworks, FactPublicAddress} // publicAddressUnknown is why the public uplink address is not said. A machine behind a NAT cannot see // its public address from itself, and asking a service outside the mesh is a dependency the mesh does not @@ -101,11 +109,24 @@ type Sources struct { // Statfs is (size, free) in bytes of the filesystem mounted at a path. Statfs func(path string) (size, free int64, err error) Now func() time.Time + // Networks is the networks on the links facing outside this machine (novox/hq ADR 0340). + Networks func() ([]outward.Network, error) } // RealSources reads this machine. func RealSources() Sources { - return Sources{Read: os.ReadFile, Readlink: os.Readlink, Run: ExecRunner, Statfs: statfs, Now: time.Now} + return Sources{Read: os.ReadFile, Readlink: os.Readlink, Run: ExecRunner, Statfs: statfs, Now: time.Now, + Networks: attachedNetworks} +} + +// attachedNetworks is this machine's networks on the links that face outside it, read the way the report +// reads those links (novox/hq ADR 0140), so the two never disagree about which links those are. +func attachedNetworks() ([]outward.Network, error) { + links, err := outward.Links("", "") + if err != nil { + return nil, err + } + return outward.Networks(links, nil) } func statfs(path string) (int64, int64, error) { @@ -193,6 +214,18 @@ func DescribeMachine(ctx context.Context, src Sources) Machine { } } + // The networks it is attached to, for the local network (novox/hq ADR 0340): every one on a link facing + // outside, or why none is said — never an empty list that reads as a machine on no network. + if src.Networks == nil { + unknown(FactNetworks, "this node-engine was given no way to read its links") + } else if networks, err := src.Networks(); err != nil { + unknown(FactNetworks, err.Error()) + } else if len(networks) == 0 { + unknown(FactNetworks, "no link facing outside this machine carries an address") + } else { + m.Networks = networks + } + unknown(FactPublicAddress, publicAddressUnknown) return m } diff --git a/internal/profile/machine_test.go b/internal/profile/machine_test.go index 9cf9e17..64c7093 100644 --- a/internal/profile/machine_test.go +++ b/internal/profile/machine_test.go @@ -10,6 +10,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/novox/mesh-host/internal/outward" ) // The parsing layer, with the machine's files given (novox/hq ADR 0017 allows a fake here and only here); @@ -40,6 +42,9 @@ func given(read map[string]string) Sources { Run: func(context.Context, string, ...string) (string, error) { return "none\n", errors.New("exited 1") }, Statfs: func(p string) (int64, int64, error) { return 1000, 400, nil }, Now: func() time.Time { return time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC) }, + Networks: func() ([]outward.Network, error) { + return []outward.Network{{Link: "eth0", Network: "192.168.1.0/24"}, {Link: "eth1", Network: "203.0.113.0/24"}}, nil + }, } } @@ -92,6 +97,7 @@ func TestMachine_aFactItCannotReadIsSaidUnknownNeverZero(t *testing.T) { src := given(map[string]string{}) src.Readlink = func(string) (string, error) { return "", os.ErrNotExist } src.Run = func(context.Context, string, ...string) (string, error) { return "", errors.New("not found") } + src.Networks = func() ([]outward.Network, error) { return nil, errors.New("cannot read the routing table") } m := DescribeMachine(context.Background(), src) raw, _ := json.Marshal(m) @@ -183,3 +189,26 @@ func TestAgainstThisMachine_machineSaysWhatItIs(t *testing.T) { raw, _ := json.MarshalIndent(m, "", " ") t.Logf("%s", raw) } + +// **The networks it is attached to are among its facts** (novox/hq ADR 0340 §7), every one — the public +// one too, which the controller judges and never counts — and a machine with none on a link facing +// outside says why rather than reporting an empty list. +func TestMachine_saysTheNetworksItIsAttachedTo(t *testing.T) { + m := DescribeMachine(context.Background(), given(files)) + raw, _ := json.Marshal(m) + var said struct { + Networks []map[string]string `json:"networks"` + } + _ = json.Unmarshal(raw, &said) + if len(said.Networks) != 2 || said.Networks[0]["link"] != "eth0" || said.Networks[0]["network"] != "192.168.1.0/24" || + said.Networks[1]["network"] != "203.0.113.0/24" { + t.Fatalf("networks said as %s", raw) + } + + src := given(files) + src.Networks = func() ([]outward.Network, error) { return nil, nil } + m = DescribeMachine(context.Background(), src) + if len(m.Networks) != 0 || !strings.Contains(m.Unknown[FactNetworks], "no link facing outside") { + t.Fatalf("a machine with no network said %v, unknown %q", m.Networks, m.Unknown[FactNetworks]) + } +}