Merge remote-tracking branch 'origin/main' into feat/335-a-busy-container-is-replaced-when-idle
mesh/merge-gate head b9a293f8c8bc merged into trunk 0e92ef3d1b21 — pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes wi…
mesh/repo-check head b9a293f8c8bc merged into trunk 0e92ef3d1b21 — pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/335-a-busy-container-is-replaced-when-idle delivered: every member is delivered

# Conflicts:
#	cmd/mesh-host/main.go
This commit is contained in:
2026-10-14 00:00:09 +02:00
34 changed files with 2146 additions and 197 deletions
+6
View File
@@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
--tools-source the repository the shared base is built from --tools-source the repository the shared base is built from
--tools-ref what of it to build (default main) --tools-ref what of it to build (default main)
--tools-go-source the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)
--tools-go-ref what of it to build (default main)
--catalog-source the catalogue REPOSITORY, for building its modules; --catalog-source the catalogue REPOSITORY, for building its modules;
--catalog is the checkout that says what they are --catalog is the checkout that says what they are
--catalog-ref what of it to build (default main) --catalog-ref what of it to build (default main)
@@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
"the repository the shared base is built from") "the repository the shared base is built from")
set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref, set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
"what of it to build (default main)") "what of it to build (default main)")
set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository,
"the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)")
set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref,
"what of it to build (default main)")
set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository, set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are") "the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref, set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
+29
View File
@@ -0,0 +1,29 @@
package main
import (
"testing"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/outward"
"github.com/novox/mesh-host/internal/profile"
)
func reportOn(networks ...outward.Network) link.Report {
return link.Report{Outward: []string{"eth0"},
Profile: profileAsReported(profile.Profile{Machine: &profile.Machine{Networks: networks}})}
}
// **A change of the networks a machine is attached to is said unasked** (novox/hq ADR 0340 §7): a port
// reached from the local network is admitted from them, so a laptop that joins another network is said at
// the next reconcile, not at the next push — and the same networks again are not news.
func TestAChangedNetworkIsSaidUnasked(t *testing.T) {
w := &adoptionWatch{}
home := reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})
w.said(home)
if w.differs(reportOn(outward.Network{Link: "eth0", Network: "192.168.1.0/24"})) {
t.Fatal("the same networks read as a change, so the machine would speak on every reconcile")
}
if !w.differs(reportOn(outward.Network{Link: "eth0", Network: "10.20.0.0/16"})) {
t.Fatal("a machine that moved to another network does not say so until it is pushed to")
}
}
+26
View File
@@ -1399,6 +1399,12 @@ func adoptionFingerprint(r link.Report) string {
// it could only speak when a declaration arrived, and a declaration cannot be composed until it // it could only speak when a declaration arrived, and a declaration cannot be composed until it
// has spoken: a machine waiting for a push that is waiting for the machine. // has spoken: a machine waiting for a push that is waiting for the machine.
parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")} parts := []string{"firewall=" + r.Firewall, "outward=" + strings.Join(r.Outward, ",")}
// **And the networks those links are on** (novox/hq ADR 0340 §7): a port reached from the local
// network is admitted from them, so a laptop that joins another network, or a lease that moves, is
// said at the next reconcile rather than at the next push or the next day.
if networks := attachedNetworksOf(r); networks != "" {
parts = append(parts, "networks="+networks)
}
// And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk // And a container waiting because it said busy, or replaced after it waited (novox/hq ADR 0335): the walk
// waiting on this machine starts its gate at the report that says it replaced. Not its output, which // waiting on this machine starts its gate at the report that says it replaced. Not its output, which
// may change on every ask. // may change on every ask.
@@ -1436,6 +1442,26 @@ func adoptionFingerprint(r link.Report) string {
return strings.Join(parts, "\n") return strings.Join(parts, "\n")
} }
// attachedNetworksOf is the networks a report's profile says the machine is attached to, as one line: the
// part of what the machine is made of that the filter is composed from (novox/hq ADR 0340).
func attachedNetworksOf(r link.Report) string {
machine, ok := r.Profile["machine"].(map[string]any)
if !ok {
return ""
}
networks, ok := machine["networks"].([]any)
if !ok {
return ""
}
said := make([]string, 0, len(networks))
for _, n := range networks {
if entry, ok := n.(map[string]any); ok {
said = append(said, fmt.Sprintf("%v %v", entry["link"], entry["network"]))
}
}
return strings.Join(said, ",")
}
// applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not // applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not
// apply and which keys its failure is said under. // apply and which keys its failure is said under.
func applyState(r link.Report) string { func applyState(r link.Report) string {
+8 -1
View File
@@ -12,7 +12,8 @@ mesh runs on:
2 a database per context `inventory`, `identity` and `licences` 2 a database per context `inventory`, `identity` and `licences`
3 those contexts' schemas mesh-controller migrate 3 those contexts' schemas mesh-controller migrate
4 the bus's certificate mesh-controller broker certificate 4 the bus's certificate mesh-controller broker certificate
5 the bus runs NATS, with the controller's account as its first user 5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it
(ADR 0333), with the controller's account as its first user
6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS) 6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS)
``` ```
@@ -26,6 +27,12 @@ rootsecrets.go). A template whose bus connection it does not recognise is refuse
naming the placeholder it looked for. The bundle that raised the previous broker was deleted with naming the placeholder it looked for. The bundle that raised the previous broker was deleted with
ADR 0331; there is one bundle, and every lab bed names it. ADR 0331; there is one bundle, and every lab bed names it.
**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container
`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in
`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus
port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once,
on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest.
Build an installer carrying a builder, and point it at this: Build an installer carrying a builder, and point it at this:
``` ```
+1 -1
View File
@@ -49,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin
} }
sort.Strings(images) sort.Strings(images)
want := []string{"mesh-controller", "nats", "postgres"} want := []string{"mesh-controller", "nats", "pgvector"}
if strings.Join(images, ",") != strings.Join(want, ",") { if strings.Join(images, ",") != strings.Join(want, ",") {
t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+ t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
"Adding one is a change to what every first node becomes, and to ADR 0006's "+ "Adding one is a change to what every first node becomes, and to ADR 0006's "+
+78 -28
View File
@@ -28,11 +28,12 @@
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The // image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
// registry prefix is the lab's; the digests are upstream's. // registry prefix is the lab's; the digests are upstream's.
// //
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host // **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container
// creates is owned by root, and the database runs as somebody else inside the container — so it // `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by
// could not write, and the container crash-looped. A named volume lets the image set up its own // the image's postgres user, its superuser password read from the module's own-secret file — so the
// ownership, and outlives the container, which is what you want for the thing holding the mesh's // store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's
// state. // registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds
// them to each other against the module's manifest).
{ {
"declaration": 1, "declaration": 1,
@@ -83,17 +84,28 @@
"boot": "enabled", "boot": "enabled",
"restart-on": ["base-filter"] "restart-on": ["base-filter"]
}, },
// The store's data is a directory owned by the image's postgres user (999), made before the
// container starts: a directory the runtime made would be root's, and the database could not
// write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory
// made so (novox/hq ADR 0338 §5).
{
"id": "store-data",
"type": "directory",
"path": "/var/lib/mesh-store",
"mode": "0700",
"owner": "999:70"
},
{ {
"id": "store", "id": "store",
"type": "container", "type": "container",
"name": "mesh-store", "name": "postgres",
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a", "image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f",
"env": { "env": {
"POSTGRES_PASSWORD": "bootstrap", "POSTGRES_PASSWORD": "bootstrap",
"PGDATA": "/var/lib/postgresql/data/pgdata" "PGDATA": "/var/lib/postgresql/data/pgdata"
}, },
"ports": ["5432:5432"], "ports": ["5432:5432"],
"volumes": ["mesh-store-data:/var/lib/postgresql/data"] "volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"]
}, },
// Over TCP, not the socket. While the store initialises it runs a temporary server on the // Over TCP, not the socket. While the store initialises it runs a temporary server on the
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action // socket ONLY, then stops it and starts the real one — so a socket check passes, the action
@@ -102,21 +114,21 @@
{ {
"id": "store-ready", "id": "store-ready",
"type": "action", "type": "action",
"in": "mesh-store", "in": "postgres",
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"], "command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"] "verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
}, },
{ {
"id": "inventory-database", "id": "inventory-database",
"type": "action", "type": "action",
"in": "mesh-store", "in": "postgres",
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"], "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"] "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
}, },
{ {
"id": "identity-database", "id": "identity-database",
"type": "action", "type": "action",
"in": "mesh-store", "in": "postgres",
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"], "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"] "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
}, },
@@ -126,20 +138,44 @@
{ {
"id": "licences-database", "id": "licences-database",
"type": "action", "type": "action",
"in": "mesh-store", "in": "postgres",
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"], "command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"] "verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
}, },
{ {
"id": "context-schemas", "id": "context-schemas",
"type": "action", "type": "action",
"command": ["docker", "run", "--rm", "--network", "container:mesh-store", "command": ["docker", "run", "--rm", "--network", "container:postgres",
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", "-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable", "-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
"migrate"], "migrate"],
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"] "verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
},
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
// /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
// catalogue's module declares. So adopting it as that module at the broker step keeps it: the
// module's built image takes the place of the upstream one below, once, on the same streams, users
// and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
// against the module's own manifest (busLayout), at the broker step and in its tests.
//
// Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
// control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
// The certificate's directory is readable by everyone because the certificate is public and the control
// plane runs as nobody; its key is written 0600 by the program that makes it.
{
"id": "bus-tls-dir",
"type": "directory",
"path": "/var/lib/mesh-broker-tls",
"mode": "0755"
},
{
"id": "bus-data-dir",
"type": "directory",
"path": "/var/lib/mesh-broker-nats",
"mode": "0700"
}, },
{ {
"id": "bus-certificate", "id": "bus-certificate",
@@ -151,49 +187,63 @@
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins // it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh // this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
// to ask an authority of. // to ask an authority of.
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right // `--user 0:0` because the directory is root's and this image runs as nobody, which is right
// for the long-running control plane and wrong for a one-shot writing into a fresh volume. // for the long-running control plane and wrong for a one-shot writing into a fresh directory.
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", "command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
"broker", "certificate", "--into", "/tls"], "broker", "certificate", "--into", "/tls"],
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls", "verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
"broker", "certificate", "--check", "--into", "/tls"] "broker", "certificate", "--check", "--into", "/tls"]
}, },
// **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
// the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
// run again after the adoption applies this bundle again, and an ordinary file would put the
// bundle's one user back over every user the mesh composed, and leave the node unable to reach its
// bus. Until the adoption the installer places the composed list over the seed (enrol.go).
{ {
"id": "bus-conf-dir", "id": "bus-conf-dir",
"type": "directory", "type": "directory",
"path": "/var/lib/mesh-bus-conf", "path": "/var/lib/nats-module/conf",
"mode": "0700" "mode": "0700"
}, },
{ {
"id": "bus-conf", "id": "bus-conf",
"type": "file", "type": "file",
"path": "/var/lib/mesh-bus-conf/nats.conf", "path": "/var/lib/nats-module/conf/nats.conf",
"mode": "0644", "mode": "0644",
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n" "create-once": true,
"content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
}, },
{ {
"id": "bus-accounts", "id": "bus-accounts",
"type": "file", "type": "file",
"path": "/var/lib/mesh-bus-conf/accounts.conf", "path": "/var/lib/nats-module/conf/accounts.conf",
"mode": "0600", "mode": "0600",
"create-once": true,
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
}, },
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
// mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned
// step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17),
// pinned upstream by that same digest: the adoption changes no server release, so it can be undone
// by putting the old image back and is said reversible. The published port is the
// node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
// module the setting that maps its declared 4222 to it (ADR 0333 §2).
{ {
"id": "broker", "id": "broker",
"type": "container", "type": "container",
"name": "mesh-broker", "name": "mesh-broker-nats",
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927", "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00",
"ports": ["5671:4222", "127.0.0.1:8222:8222"], "ports": ["5671:4222", "127.0.0.1:8222:8222"],
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"], "volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
"args": ["-c", "/etc/nats/nats.conf", "-js"] "args": ["-c", "/etc/nats/nats.conf", "-js"]
}, },
{ {
"id": "broker-ready", "id": "broker-ready",
"type": "action", "type": "action",
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"], "command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"] "verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"]
}, },
{ {
"id": "control-plane", "id": "control-plane",
@@ -202,7 +252,7 @@
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace", "image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
"network": "host", "network": "host",
"args": ["serve"], "args": ["serve"],
"volumes": ["mesh-broker-tls:/broker-tls:ro"], "volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
"env": { "env": {
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable", "MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable", "MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
+5 -1
View File
@@ -174,6 +174,10 @@ type Options struct {
// ToolsSource is where the shared base is built from. Everything with code of its own // ToolsSource is where the shared base is built from. Everything with code of its own
// compiles against it, so it is the first thing the mesh builds for itself. // compiles against it, so it is the first thing the mesh builds for itself.
ToolsSource Source ToolsSource Source
// ToolsGoSource is where the Go toolchain is built from (`mesh-tools-go`, ADR 0142): every Go
// bundle compiles in it, and genesis's own modules have Go code (issue 616). Empty means the
// repository beside ToolsSource named mesh-tools-go, as the mesh keeps it.
ToolsGoSource Source
// CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue // CatalogSource is where the catalogue REPOSITORY is, for building its modules. The catalogue
// CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it. // CHECKOUT (Catalogue, above) says what a module is; this is where a builder clones it.
CatalogSource Source CatalogSource Source
@@ -734,7 +738,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
// ---- broker --------------------------------------------------------------------------- // ---- broker ---------------------------------------------------------------------------
say("broker — the foundation's bus, adopted as the nats module: one bus, not two") say("broker — the foundation's bus, adopted as the nats module: one bus, not two")
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil { if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, d.Fetch, say); err != nil {
return result, failed(StepBroker, err) return result, failed(StepBroker, err)
} }
+72 -18
View File
@@ -1,13 +1,16 @@
package bootstrap package bootstrap
import ( import (
"bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"strings"
) )
// BuilderModule is the module that lets a mesh produce anything at all. // BuilderModule is the module that lets a mesh produce anything at all: the build agent, which holds
const BuilderModule = "builder" // the node seat `node-build-agent` (novox/hq ADR 0190). The one-holder `builder` module it replaced
// left the catalogue on 2026-10-03, and genesis went on asking for it (issue 596).
const BuilderModule = "build-agent"
// BuilderRepository is what its image is called in this mesh's own registry. // BuilderRepository is what its image is called in this mesh's own registry.
const BuilderRepository = "mesh-builder" const BuilderRepository = "mesh-builder"
@@ -50,10 +53,10 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
manifest, err := readManifest(o.Catalogue, BuilderModule) manifest, err := readManifest(o.Catalogue, BuilderModule)
if err != nil { if err != nil {
return out, fmt.Errorf("%w\n"+ return out, fmt.Errorf("%w\n"+
"This is the manifest that makes the builder an ordinary module. Without it the mesh "+ "This is the manifest that makes the build agent an ordinary module. Without it the mesh "+
"has the image and no way to run it, so nothing can be built here", err) "has the image and no way to run it, so nothing can be built here", err)
} }
pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule) pinned, places, err := pinTheCarriedBuilder(manifest, published.Reference)
if err != nil { if err != nil {
return out, err return out, err
} }
@@ -66,21 +69,72 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
} }
// Before the push, so the account is in the declaration the machine receives rather than in // Before the push, so the account is in the declaration the machine receives rather than in
// the one after it. A builder that arrives without its credential starts, finds nothing it may // the one after it. A build agent that arrives without its credential starts, finds nothing it
// read, and waits — which looks exactly like a builder with no work. // may read, and waits — which looks exactly like an agent with no work. The same act as any
account := o.Node + "-" + BuilderModule // module's account (`module issue`), which a re-run repeats harmlessly.
if _, err := control.tell(ctx, "builder", "issue", account, "--node", o.Node); err != nil { if _, err := control.tell(ctx, "module", "issue", BuilderModule, "--node", o.Node); err != nil {
// Said and carried on. An account that already exists is the ordinary case on a re-run, return out, fmt.Errorf("the build agent has no account on the bus, so it can take no work: %w", err)
// and the push below is what makes either state true on the machine.
if !strings.Contains(err.Error(), "already") {
return out, fmt.Errorf("the builder has no broker account, so it can take no work: %w", err)
}
say(" broker account " + account + " — already issued")
} else {
out.Account = true
say(" broker account " + account + ", scoped to what it consumes and emits")
} }
out.Account = true
say(" bus account " + BuilderModule + " on " + o.Node + ", scoped to what it consumes and emits")
out.Installed.Pushed, err = pushNode(ctx, o, control, say) out.Installed.Pushed, err = pushNode(ctx, o, control, say)
return out, err return out, err
} }
// pinTheCarriedBuilder names the image this installer carried and published in the build agent's
// manifest, in place of the artifact the mesh would build for it.
//
// **The one module genesis cannot have built** (novox/hq ADR 0073): the build agent is what builds,
// and at this step the mesh has no build agent. The catalogue's manifest names its server container by
// the artifact it builds (`"artifact": "server"`, an image built from mesh-controller's
// cmd/mesh-builder); the image this installer carries is that build, made from the same program. So
// every container built from that image artifact runs the published reference instead, and the rest
// of the manifest — its build, its seat, its requirements — is the catalogue's as written. The mesh
// rebuilds it from the catalogue like any module from then on.
//
// A manifest still carrying the old placeholder digest is pinned the old way. One naming neither an
// image artifact nor a placeholder is refused: there would be nothing to put the carried image in.
func pinTheCarriedBuilder(manifest []byte, reference string) ([]byte, int, error) {
if bytes.Contains(manifest, []byte(placeholderDigest)) {
return pinPlaceholder(manifest, reference, BuilderModule)
}
var m map[string]any
if err := json.Unmarshal(manifest, &m); err != nil {
return nil, 0, fmt.Errorf("the %s module's manifest is not readable: %w", BuilderModule, err)
}
images := map[string]bool{}
if build, ok := m["build"].(map[string]any); ok {
artifacts, _ := build["artifacts"].([]any)
for _, a := range artifacts {
if art, ok := a.(map[string]any); ok && art["kind"] == "image" {
if name, ok := art["name"].(string); ok {
images[name] = true
}
}
}
}
places := 0
resources, _ := m["resources"].([]any)
for _, r := range resources {
res, ok := r.(map[string]any)
if !ok || res["type"] != "container" {
continue
}
if name, ok := res["artifact"].(string); ok && images[name] {
delete(res, "artifact")
res["image"] = reference
places++
}
}
if places == 0 {
return nil, 0, fmt.Errorf(
"the %s module's manifest runs no container from an image it builds, so there is nowhere to "+
"put the builder this installer carried and published (%s)", BuilderModule, reference)
}
pinned, err := json.MarshalIndent(m, "", " ")
if err != nil {
return nil, 0, err
}
return append(pinned, '\n'), places, nil
}
+71 -21
View File
@@ -2,6 +2,7 @@ package bootstrap
import ( import (
"context" "context"
"encoding/json"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -37,27 +38,6 @@ func recording(t *testing.T) (*controlRecorder, controlPlane) {
return c, controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second} return c, controlPlane{container: "temp-mesh-controller", run: c.run, timeout: time.Second}
} }
func TestTheBuildersPackageRegistryPortIsASettingAndNotTheManifest(t *testing.T) {
c, control := recording(t)
o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second}
if _, err := installModule(context.Background(), o, control, BuilderModule,
[]byte(theBuildersBinding), quietly); err != nil {
t.Fatal(err)
}
if got := c.settings["builder-settings.json"]; got != `{"serves":{"port":3100}}` {
t.Errorf("the builder was told %q about the package registry's port", got)
}
// The manifest reaches the mesh as the catalogue wrote it. A port rewritten into it here is a
// port the next registration from the catalogue silently takes back.
if got := c.settings["builder-module.json"]; got != theBuildersBinding {
t.Errorf("the installer changed the builder's manifest:\n%s", got)
}
set, push := c.index("settings set builder"), c.index("push anchor")
if set < 0 || push < 0 || set > push {
t.Fatalf("the port was not set before the push that raises the builder: %v", c.told)
}
}
func TestTheBuilderOnTheDefaultPackagesPortIsToldNothing(t *testing.T) { func TestTheBuilderOnTheDefaultPackagesPortIsToldNothing(t *testing.T) {
c, control := recording(t) c, control := recording(t)
o := Options{Node: "anchor", Wait: time.Second} o := Options{Node: "anchor", Wait: time.Second}
@@ -167,3 +147,73 @@ func TestAForgeManifestTheCatalogueDoesNotHaveIsNamed(t *testing.T) {
t.Errorf("a missing forge manifest was not explained: %v", err) t.Errorf("a missing forge manifest was not explained: %v", err)
} }
} }
// theCataloguesModule is the catalogue's own manifest of a module, never a copy: the clone beside a merge
// check (MESH_CHECK_BESIDE), where it must be; else a checkout beside this one; else the test skips aloud.
func theCataloguesModule(t *testing.T, module string) []byte {
t.Helper()
inside := filepath.Join("mesh-catalog", "modules", module, "module.json")
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
raw, err := os.ReadFile(filepath.Join(root, inside))
if err != nil {
t.Fatalf("the catalogue beside this check (MESH_CHECK_BESIDE=%s) has no %s: %v", root, inside, err)
}
return raw
}
raw, err := os.ReadFile(filepath.Join("..", "..", "..", inside))
if err != nil {
t.Skipf("NOT JUDGED: no catalogue beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
}
return raw
}
// **Genesis installs the build agent the catalogue holds** (novox/hq issue 596, ADR 0190): the
// one-holder `builder` left the catalogue on 2026-10-03, and genesis stopped at step 13 asking for it.
// The catalogue's own manifest is read, and the image the installer carried is put where the manifest
// names the image it builds.
func TestGenesisInstallsTheCataloguesBuildAgent(t *testing.T) {
manifest := theCataloguesModule(t, BuilderModule)
const reference = "127.0.0.1:5000/mesh-builder@sha256:" + "4444444444444444444444444444444444444444444444444444444444444444"
pinned, places, err := pinTheCarriedBuilder(manifest, reference)
if err != nil {
t.Fatal(err)
}
var m struct {
Module string `json:"module"`
Claims []struct {
Name string `json:"name"`
} `json:"claims"`
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(pinned, &m); err != nil {
t.Fatal(err)
}
if m.Module != BuilderModule || len(m.Claims) == 0 || m.Claims[0].Name != "node-build-agent" {
t.Fatalf("genesis installs %q claiming %v; the build agent holds node-build-agent (ADR 0190)", m.Module, m.Claims)
}
running := 0
for _, r := range m.Resources {
if r["type"] != "container" {
continue
}
if _, still := r["artifact"]; still {
t.Errorf("the container %v still names an artifact nothing has built yet", r["name"])
}
if r["image"] == reference {
running++
}
}
if places != 1 || running != 1 {
t.Errorf("the carried builder runs in %d container(s) (%d place(s) said); the build agent runs one", running, places)
}
}
// A manifest that runs nothing from an image it builds has nowhere to put the carried builder, and is
// refused rather than registered running nothing.
func TestABuildAgentManifestWithNowhereForTheCarriedBuilderIsRefused(t *testing.T) {
_, _, err := pinTheCarriedBuilder([]byte(`{"module":"build-agent","resources":[{"id":"server","type":"container","name":"x","artifact":"server"}],"build":{"artifacts":[{"name":"server","kind":"bundle"}]}}`),
"127.0.0.1:5000/mesh-builder@sha256:"+strings.Repeat("4", 64))
if err == nil || !strings.Contains(err.Error(), "nowhere to put") {
t.Fatalf("a manifest with no image artifact was pinned: %v", err)
}
}
+179
View File
@@ -0,0 +1,179 @@
package bootstrap
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"os"
"regexp"
"strconv"
"strings"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// Adopting the bus is the mesh's planned step for replacing it (novox/hq ADR 0333 §3, ADR 0236).
//
// The adoption replaces the bus's image once: the module's built image takes the place of the upstream
// one the bundle raised, on the same streams, users and certificate. The controller refuses a push that
// would replace the bus — "a planned step: `bus upgrade --why …` snapshots its streams first and checks
// them after" — so genesis stopped at step 18 (issue 590). The installer takes that step: it snapshots the
// streams itself, because at genesis no backup holder exists to be asked, and hands the snapshot to
// `bus upgrade`, which sends the machine, opens `bus-maintenance`, and ends the step done only once the
// bus answers healthy again — every stream, every durable consumer, a round trip (H-bus). The installer
// waits for that end.
// adoptionWhy is what the step is recorded as, for the record of hand acts and the bus's step.
const adoptionWhy = "genesis adopts its bus as the nats module (ADR 0333)"
// Where the installer keeps the streams' snapshot before the adoption, and the credential the snapshot
// program reads while it runs. Beside genesis's own credentials; variables so a test writes elsewhere.
var (
busSnapshotFile = "/var/lib/mesh-genesis/bus-before-adoption.tar"
busSnapshotCredential = "/var/lib/mesh-genesis/bus-snapshot.credential"
busCertificateFile = "/var/lib/mesh-broker-tls/tls.crt"
busControllerSecret = BusControllerFile
// busStepWithin bounds the wait for the controller to say the step ended: its own bound (15 min,
// mesh-controller busStepBound), and its doctor's next look after that.
busStepWithin = 17 * time.Minute
)
// adoptTheBus sends the nats module to this machine through the bus's planned step. A bus already on the
// module's image is the module's: it is sent like any module, and nothing is replaced.
func adoptTheBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container,
say func(string)) error {
if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image {
say(" already adopted " + bus.Name + " — it runs the module's image")
_, err := pushNode(ctx, o, control, say)
return err
}
image, err := theModulesBusImage(ctx, o, control)
if err != nil {
return err
}
if err := snapshotTheBus(ctx, o, control, image, say); err != nil {
return err
}
say(" restarting " + bus.Name + " — once, onto the " + BusModule + " module's image, as the bus's " +
"planned step; its streams, users and certificate stay where they are (novox/hq ADR 0333 §3, ADR 0236)")
if _, err := control.within(2*o.Wait).tell(ctx, "bus", "upgrade", "--why", adoptionWhy,
"--reversible", "--snapshot-taken", busSnapshotFile); err != nil {
return fmt.Errorf("the bus's planned step was refused, so the bus is not replaced: %w", err)
}
return nil
}
// meshBuiltImage is an image the mesh built, as it records it: `artifact-store://<module>/<artifact>@sha256:…`.
var meshBuiltImage = regexp.MustCompile(`artifact-store://([a-z0-9][a-z0-9-]*/[a-z0-9][a-z0-9-]*@sha256:[0-9a-f]{64})`)
// theModulesBusImage is the nats module's server image, as the build just made it, pulled from this
// mesh's registry: the image that carries the snapshot program.
func theModulesBusImage(ctx context.Context, o Options, control controlPlane) (string, error) {
said, err := control.tell(ctx, "builds", BusModule, "-n", "1")
if err != nil {
return "", err
}
for _, m := range meshBuiltImage.FindAllStringSubmatch(said, -1) {
if strings.HasPrefix(m[1], BusModule+"/") {
return strings.TrimSuffix(o.Registry, "/") + "/" + m[1], nil
}
}
return "", fmt.Errorf("the mesh says no image it built for the %s module, so its snapshot program cannot "+
"be run before the bus is replaced:\n%s", BusModule, indent(strings.TrimSpace(said)))
}
// snapshotTheBus writes every stream of the running bus to busSnapshotFile with the module's own snapshot
// program (mesh-catalog modules/nats, ADR 0235), as the controller's account, pinned to the bus's
// certificate. The credential is written for the program's one run and removed after it.
func snapshotTheBus(ctx context.Context, o Options, control controlPlane, image string, say func(string)) error {
password, err := readCredentialFile(busControllerSecret)
if err != nil {
return fmt.Errorf("the controller's bus credential is not at %s, so the streams cannot be snapshotted "+
"before the bus is replaced: %w", busControllerSecret, err)
}
fingerprint, err := certificateFingerprint(busCertificateFile)
if err != nil {
return err
}
port := o.Ports.orDefaults().Bus
server := "tls://127.0.0.1:" + strconv.Itoa(port)
credential, err := json.Marshal(map[string]string{"url": server, "fingerprint": fingerprint,
"user": busControllerUser, "password": password})
if err != nil {
return err
}
if err := os.WriteFile(busSnapshotCredential, credential, 0o600); err != nil {
return fmt.Errorf("the snapshot's credential could not be written: %w", err)
}
defer os.Remove(busSnapshotCredential)
script := "docker run --rm -i --network host --entrypoint /usr/local/bin/mesh-nats-snapshot " + image +
" snapshot --server " + server + " --credential - < " + busSnapshotCredential + " > " + busSnapshotFile +
".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile
running, cancel := context.WithTimeout(ctx, busStepWithin)
defer cancel()
if out, err := control.run(running, "sh", "-c", script); err != nil {
_, _ = control.run(ctx, "rm", "-f", busSnapshotFile+".partial")
return fmt.Errorf("the bus's streams could not be snapshotted, so the bus is not replaced: %w\n%s",
err, indent(strings.TrimSpace(out)))
}
say(" snapshotted every stream of the bus, at " + busSnapshotFile + " (its module's snapshot program)")
return nil
}
// certificateFingerprint is the pin of the bus's certificate: sha256 of its DER, as every machine pins it.
func certificateFingerprint(path string) (string, error) {
raw, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("the bus's certificate is not at %s, so it cannot be pinned for the snapshot: %w", path, err)
}
block, _ := pem.Decode(raw)
if block == nil {
return "", fmt.Errorf("%s holds no PEM certificate", path)
}
sum := sha256.Sum256(block.Bytes)
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
// waitForTheBusStep waits until the controller says the bus's planned step ended: done once the bus
// answered healthy — every stream, every durable consumer, a round trip — or failed, with its snapshot as
// the way back. The step's own bound is the controller's; this waits a little past it.
func waitForTheBusStep(ctx context.Context, o Options, control controlPlane, say func(string)) error {
deadline := time.Now().Add(busStepWithin)
var last string
for {
said, err := control.tell(ctx, "bus")
if err == nil {
for _, line := range strings.Split(said, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "last step ") {
last = strings.TrimSpace(line)
}
}
switch {
case strings.Contains(last, ": done at "):
say(" bus step done " + last)
return nil
case strings.Contains(last, ": failed at "):
return fmt.Errorf("the bus's planned step ended failed: %s\n%s", last, indent(strings.TrimSpace(said)))
}
}
if !time.Now().Before(deadline) {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(answerEvery):
}
}
if last == "" {
return errors.New("the controller says no bus step after the adoption was sent; `mesh-controller bus` says how it stands")
}
return fmt.Errorf("the bus's planned step did not end within %s: %s", busStepWithin, last)
}
+135
View File
@@ -0,0 +1,135 @@
package bootstrap
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
)
// Defends novox/hq issue 590 at its broker step (ADR 0333 §3, ADR 0236): the adoption replaces the bus's
// image, and the mesh replaces its bus only as a planned step. A plain push was refused — "sending anchor
// would replace the bus … a planned step: `bus upgrade --why …`" — and genesis stopped at step 18.
func TestTheBrokerStepAdoptsTheBusThroughItsPlannedStep(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
upgraded := false
var credential []byte
runtime := &asked{}
agrees := aMeshThatAgrees(nil)
runtime.answer = func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case name == "sh" && strings.Contains(joined, "mesh-nats-snapshot"):
credential, _ = os.ReadFile(busSnapshotCredential) // as the program reads it
case strings.Contains(joined, "bus upgrade --why"):
upgraded = true
case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
if upgraded {
return theModulesImage + "\n", nil
}
return bundleImage + "\n", nil
}
return agrees(name, args)
}
_, fetch := aBusThatAdopts()
o := installing(t, catalogue)
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
o.Wait = 5 * time.Second
said, err := busStepWith(t, o, runtime, fetch)
if err != nil {
t.Fatal(err)
}
snapshot, upgrade := -1, -1
for i, c := range runtime.commands {
switch {
case strings.HasPrefix(c, "sh -c") && strings.Contains(c, "mesh-nats-snapshot"):
snapshot = i
for _, want := range []string{"127.0.0.1:5000/nats/server@sha256:" + strings.Repeat("4", 64),
"snapshot --server tls://127.0.0.1:5671 --credential - < " + busSnapshotCredential,
"> " + busSnapshotFile + ".partial && mv " + busSnapshotFile + ".partial " + busSnapshotFile} {
if !strings.Contains(c, want) {
t.Errorf("the snapshot is not taken with %q: %s", want, c)
}
}
case strings.Contains(c, "/mesh-controller bus upgrade"):
upgrade = i
if want := "bus upgrade --why " + adoptionWhy + " --reversible --snapshot-taken " + busSnapshotFile; !strings.HasSuffix(c, want) {
t.Errorf("the planned step is asked as %q; want it to end %q", c, want)
}
case strings.HasSuffix(c, "/mesh-controller push anchor"):
t.Errorf("the bus's module was sent by a plain push, which the mesh refuses for the bus: %v", runtime.commands)
}
}
if snapshot < 0 || upgrade < 0 || snapshot > upgrade {
t.Fatalf("the streams were not snapshotted before the bus's planned step (snapshot %d, upgrade %d): %v",
snapshot, upgrade, runtime.commands)
}
var cred map[string]string
if err := json.Unmarshal(credential, &cred); err != nil {
t.Fatalf("the snapshot program was given no credential: %v", err)
}
fingerprint, _ := certificateFingerprint(busCertificateFile)
if cred["user"] != "controller" || cred["password"] != "BUS-PW" || cred["fingerprint"] != fingerprint ||
cred["url"] != "tls://127.0.0.1:5671" {
t.Errorf("the snapshot's credential is %v", cred)
}
if _, err := os.Stat(busSnapshotCredential); !os.IsNotExist(err) {
t.Error("the snapshot's credential was left on the machine")
}
text := strings.Join(said, "\n")
for _, want := range []string{"snapshotted every stream", "restarting mesh-broker-nats — once",
"bus answers mesh-broker-nats", "bus step done last step 1", "adopted mesh-broker-nats"} {
if !strings.Contains(text, want) {
t.Errorf("the step did not say %q:\n%s", want, text)
}
}
}
// A planned step the controller ends failed is the broker step failing, with the controller's words.
func TestABusStepThatEndsFailedIsRefused(t *testing.T) {
aGenesisBusOnDisk(t)
runtime := &asked{answer: func(name string, args []string) (string, error) {
return "last step 1: not known → 53e51864 on anchor by installer (x): failed at 2026-10-13 11:59; snapshot s\n" +
" stream KV_x has no messages\n", nil
}}
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
err := waitForTheBusStep(context.Background(), Options{}, control, quietly)
if err == nil || !strings.Contains(err.Error(), "failed at") || !strings.Contains(err.Error(), "KV_x") {
t.Fatalf("a failed bus step was not refused with its words: %v", err)
}
}
// The bundle's bus runs the release the module's image is built on, pinned by the same digest, so the
// adoption changes no server release and is reversible as the step says (ADR 0333 §3).
func TestTheBundlesBusRunsTheReleaseTheModulesImageIsBuiltOn(t *testing.T) {
var m struct {
Build struct {
On []struct{ Arg, Image string } `json:"on"`
} `json:"build"`
}
if err := json.Unmarshal(theCataloguesModule(t, BusModule), &m); err != nil {
t.Fatal(err)
}
base := ""
for _, on := range m.Build.On {
if on.Arg == "NATS_BASE" {
base = on.Image
}
}
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
if got := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image; got != base {
t.Errorf("the bundle raises the bus on %s; the nats module's image is built on %s", got, base)
}
}
+16 -2
View File
@@ -311,6 +311,17 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) {
for name, path := range m.OwnSecrets { for name, path := range m.OwnSecrets {
secretAt[path] = name secretAt[path] = name
} }
// **A file the module itself declares is its own, not a secret** (novox/hq issue 597). The
// controller reads its merge window's settings from a file its manifest writes from its settings
// (MESH_MERGE_WINDOW_FILE, ADR 0276): the mesh writes it with the module's declaration, and there
// is nothing for genesis to carry in. Only a `…_FILE` naming neither an own-secret nor a file of
// the module is refused.
declared := map[string]bool{}
for _, r := range m.Resources {
if r.Type == "file" && r.Path != "" {
declared[r.Path] = true
}
}
wanted := map[string]string{} wanted := map[string]string{}
for _, r := range m.Resources { for _, r := range m.Resources {
@@ -333,10 +344,13 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) {
on = from on = from
} }
secret, named := secretAt[on] secret, named := secretAt[on]
if !named && declared[on] {
continue
}
if !named { if !named {
return nil, fmt.Errorf( return nil, fmt.Errorf(
"the %s module's container reads %s from %s, and no own-secret of that "+ "the %s module's container reads %s from %s, and neither an own-secret nor "+
"module writes that file.\n"+ "a file of that module writes that file.\n"+
"So the mesh would seal nothing there and the control plane would find "+ "So the mesh would seal nothing there and the control plane would find "+
"an empty file where a connection string has to be. The manifest has to "+ "an empty file where a connection string has to be. The manifest has to "+
"name the two ends the same, directly or through a mount", "name the two ends the same, directly or through a mount",
+11 -6
View File
@@ -94,10 +94,10 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
switch mine, err := identity.Load(where); { switch mine, err := identity.Load(where); {
case err == nil && mine.Node == o.Node: case err == nil && mine.Node == o.Node:
say(" already enrolled " + o.Node + " — " + where) say(" already enrolled " + o.Node + " — " + where)
// **Enrolled is not adopted** (novox/hq ADR 0331 §5). This run's apply wrote the bundle's // **Enrolled is not adopted** (novox/hq ADR 0331 §5). Until the bus is the nats module, its
// user list again — the controller's entry and nobody else's — over the list placed when // user list is the installer's to place, so it is placed again, before anything dials the
// the machine enrolled. Until the bus is the nats module, that list is still the // bus as a node: the controller's composition is the one that is right, whatever an earlier
// installer's to place, so it is placed again, before anything dials the bus as a node. // run left there. The bundle seeds the list only where none is (create-once, ADR 0333 §4).
if err := placeUntilAdopted(ctx, control, say); err != nil { if err := placeUntilAdopted(ctx, control, say); err != nil {
return out, err return out, err
} }
@@ -168,9 +168,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
// configuration includes, and the container that reads it. The installer raised them, so it is the // configuration includes, and the container that reads it. The installer raised them, so it is the
// one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the // one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the
// nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5). // nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5).
//
// **The module's path and the module's container** (novox/hq ADR 0333 §4): the bundle raises the bus
// where the `nats` module keeps it, so the list placed here is the one the module goes on writing, in
// the same file, read by the same container. The tests hold both to the bundle and to the module's
// manifest (`bus-users` and its server container).
const ( const (
busAccounts = "/var/lib/mesh-bus-conf/accounts.conf" busAccounts = "/var/lib/nats-module/conf/accounts.conf"
busContainer = "mesh-broker" busContainer = "mesh-broker-nats"
) )
// placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes // placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes
+3 -3
View File
@@ -315,8 +315,8 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
placed, enrol := []int{}, -1 placed, enrol := []int{}, -1
for i, c := range runtime.commands { for i, c := range runtime.commands {
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") && if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker-nats") &&
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") { strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") {
placed = append(placed, i) placed = append(placed, i)
} }
if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") { if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
@@ -365,7 +365,7 @@ func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *tes
placed := false placed := false
for _, c := range runtime.commands { for _, c := range runtime.commands {
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") && if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") &&
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker") { strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker-nats") {
placed = true placed = true
} }
} }
+82
View File
@@ -180,3 +180,85 @@ func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) {
} }
} }
} }
// theStepNineSecretsOf is what step 9 delivers for a process-form controller manifest: its genesis
// container made, pinned, and its secrets read from that container's environment.
func theStepNineSecretsOf(t *testing.T, manifest []byte) (map[string]string, error) {
t.Helper()
form, err := processFormOf(manifest)
if err != nil || !form.Found {
t.Fatalf("the manifest has no process form: %v", err)
}
genesis, err := genesisForm(manifest, form, builtImage)
if err != nil {
t.Fatal(err)
}
pinned, _, err := pinImage(genesis, builtImage, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule)
if err != nil {
t.Fatal(err)
}
return secretsByVariableIn(pinned)
}
// **A file the controller's manifest writes itself is not a secret** (novox/hq issue 597). The merge
// window's settings reach the controller in a file its manifest declares from its settings
// (MESH_MERGE_WINDOW_FILE, ADR 0276); genesis refused it as a secret nobody writes, and stopped at
// step 11 of 24.
func TestAFileTheControllersManifestWritesIsNotASecret(t *testing.T) {
manifest := strings.Replace(processFormManifest,
`{"id": "controller", "type": "process",`,
`{"id": "merge-window", "type": "file", "path": "${dir:mesh-state}/merge-window.json", "mode": "0600",
"owner": "mesh-controller", "content": "{\"merge-window\": \"${setting:merge-window}\"}\n"},
{"id": "controller", "type": "process",`, 1)
manifest = strings.Replace(manifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`,
`"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus",
"MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1)
if !strings.Contains(manifest, "MESH_MERGE_WINDOW_FILE") || !strings.Contains(manifest, `"id": "merge-window"`) {
t.Fatal("the fixture was not changed")
}
wanted, err := theStepNineSecretsOf(t, []byte(manifest))
if err != nil {
t.Fatalf("step 9 refuses a file the manifest writes itself: %v", err)
}
if _, asked := wanted["MESH_MERGE_WINDOW"]; asked {
t.Errorf("the merge window's settings are delivered as a secret: %v", wanted)
}
if wanted["MESH_STORE_INVENTORY"] != "inventory" || wanted["MESH_BUS_NATS"] != "bus" {
t.Errorf("step 9 delivers %v; the store's and the bus's secrets are still wanted", wanted)
}
}
// And a `…_FILE` that neither an own-secret nor a file of the module writes is still refused.
func TestAFileNothingOfTheControllerWritesIsStillRefused(t *testing.T) {
manifest := strings.Replace(processFormManifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`,
`"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus",
"MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1)
if _, err := theStepNineSecretsOf(t, []byte(manifest)); err == nil || !strings.Contains(err.Error(), "MESH_MERGE_WINDOW_FILE") {
t.Fatalf("a file nothing writes was accepted, or refused without its name: %v", err)
}
}
// The controller's own manifest, as the mesh runs it, passes step 9 (issue 597): in a merge check the
// node-engine's check has the controller cloned beside it (MESH_CHECK_BESIDE), where it must be; else a
// checkout beside this one; else it skips aloud.
func TestTheControllersOwnManifestPassesStepNine(t *testing.T) {
var manifest []byte
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
raw, err := os.ReadFile(filepath.Join(root, "mesh-controller", "module.json"))
if err != nil {
t.Fatalf("the controller is not beside this check in MESH_CHECK_BESIDE=%s: %v", root, err)
}
manifest = raw
} else if raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-controller", "module.json")); err == nil {
manifest = raw
} else {
t.Skip("NOT JUDGED: no controller beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
}
wanted, err := theStepNineSecretsOf(t, manifest)
if err != nil {
t.Fatalf("genesis's step 9 refuses the controller the mesh runs: %v", err)
}
if wanted["MESH_STORE_INVENTORY"] != "inventory" {
t.Errorf("step 9 delivers %v", wanted)
}
}
@@ -0,0 +1,65 @@
package bootstrap
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
)
// Genesis seals the build agent the forge it raised (novox/hq ADR 0336 §3, issue 121): the agent's
// npm-package-registry requirement is optional and binds only once the gitea module is assigned, after
// the base, so until then the packages step's genesis document is how the sdk and base steps reach the
// forge.
func TestThePackagesStepSealsTheGenesisDocumentToTheBuildAgent(t *testing.T) {
if BuilderModule != "build-agent" {
t.Fatalf("genesis installs %q; the document is the build agent's", BuilderModule)
}
c, control := recording(t)
o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second}
doc := genesisRegistry(o.Ports.orDefaults(), "builder-password")
if err := deliverGenesisRegistry(context.Background(), o, control, doc, quietly); err != nil {
t.Fatal(err)
}
carried := c.settings["accepting-"+GenesisRegistrySecret]
var got GenesisRegistry
if err := json.Unmarshal([]byte(carried), &got); err != nil {
t.Fatalf("the genesis document carried is not JSON: %q", carried)
}
want := GenesisRegistry{Scheme: "http", At: "127.0.0.1", Port: 3100, NpmPath: "/api/packages/novox/npm/",
As: "mesh-builder", Password: "builder-password"}
if got != want {
t.Fatalf("the genesis document says %+v, want %+v", got, want)
}
accept := c.index("secret accept anchor build-agent genesis-package-registry --from /accepting-genesis-package-registry")
push := c.index("push anchor")
if accept < 0 || push < 0 || accept > push {
t.Fatalf("the genesis document was not sealed to the build agent before the push: %v", c.told)
}
}
// What the retired builder was given and the build agent declares neither of: an `npm-password`
// own-secret and a `serves` setting for its carried binding (ADR 0336 §3).
func TestNoStepGivesTheBuildAgentAServesSettingOrAnNpmPassword(t *testing.T) {
for _, module := range []string{BuilderModule} {
c, control := recording(t)
o := Options{Node: "anchor", Ports: FoundationPorts{Packages: 3100}, Wait: time.Second}
if _, err := installModule(context.Background(), o, control, module, []byte(`{"module":"`+module+`"}`), quietly); err != nil {
t.Fatal(err)
}
if got, set := c.settings[module+"-settings.json"]; set {
t.Errorf("%s was given a setting on a node with a non-default packages port: %s", module, got)
}
if err := deliverGenesisRegistry(context.Background(), o, control,
genesisRegistry(o.Ports.orDefaults(), "p"), quietly); err != nil {
t.Fatal(err)
}
for _, told := range c.told {
if strings.Contains(told, "npm-password") || strings.Contains(told, "serves") {
t.Errorf("genesis told the controller %q", told)
}
}
}
}
@@ -0,0 +1,89 @@
package bootstrap
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
)
// Defends novox/hq issue 616: genesis's base step builds the toolchain of every language a module
// genesis installs is compiled in. It built only mesh-tools, and the store's module, with Go code,
// stopped genesis at step 17: "this mesh holds no copy of it. Build mesh-tools-go first".
// genesisModules is every catalogue module genesis installs on its own, read from the steps that
// install them: the registry, the build agent, the store, the bus, the vault, the catalogue and the
// filter (extras are the operator's choice and come after the base too, but are not genesis's own).
var genesisModules = []string{RegistryModule, BuilderModule, "postgres", BusModule, "mesh-vault", "mesh-catalog", "nftables"}
// Every genesis module's bundles are compiled in a toolchain the base step builds, read from the real
// catalogue's manifests: a module that gains a language genesis does not build fails here, not at a
// genesis.
func TestTheBaseStepBuildsEveryToolchainAGenesisModuleIsCompiledIn(t *testing.T) {
built := map[string]bool{}
for _, tc := range genesisToolchains {
built[tc.Language] = true
}
for _, module := range genesisModules {
var m struct {
Build struct {
Artifacts []struct {
Name, Kind, Language string
} `json:"artifacts"`
} `json:"build"`
}
if err := json.Unmarshal(theCataloguesModule(t, module), &m); err != nil {
t.Fatalf("%s: %v", module, err)
}
for _, a := range m.Build.Artifacts {
if a.Kind != "bundle" {
continue // an image builds on the bases its Dockerfile names, pinned upstream
}
if !built[a.Language] {
t.Errorf("%s's %q is a %s bundle, and genesis's base step builds no %s toolchain",
module, a.Name, a.Language, a.Language)
}
}
}
}
// And the base step asks for each toolchain's build: the base, then the Go toolchain beside it.
func TestTheBaseStepBuildsTheGoToolchainBesideTheBase(t *testing.T) {
runtime := &asked{answer: aMeshThatAgrees(nil)}
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
o := Options{ToolsSource: Source{Repository: "git://192.0.2.30/mesh-tools.git"}}
if err := BuildBase(context.Background(), o, control, quietly); err != nil {
t.Fatal(err)
}
var builds []string
for _, c := range runtime.commands {
if strings.Contains(c, "/mesh-controller build ") {
builds = append(builds, c[strings.Index(c, "build "):])
}
}
want := []string{
"build git://192.0.2.30/mesh-tools.git --ref main --wait 1200s",
"build git://192.0.2.30/mesh-tools-go.git --ref main --wait 1200s",
}
if strings.Join(builds, "\n") != strings.Join(want, "\n") {
t.Fatalf("the base step built\n%s\nand not\n%s", strings.Join(builds, "\n"), strings.Join(want, "\n"))
}
}
func TestTheGoToolchainIsAsGivenOrBesideTheBase(t *testing.T) {
for _, c := range []struct{ tools, given, want string }{
{"http://forge/novox/mesh-tools.git", "", "http://forge/novox/mesh-tools-go.git"},
{"http://forge/novox/mesh-tools", "", "http://forge/novox/mesh-tools-go"},
{"http://forge/other/base.git", "http://forge/other/go.git", "http://forge/other/go.git"},
} {
got, err := toolsGoSource(Options{ToolsSource: Source{Repository: c.tools}, ToolsGoSource: Source{Repository: c.given}})
if err != nil || got.Repository != c.want {
t.Errorf("tools %s, given %q: %q %v; want %s", c.tools, c.given, got.Repository, err, c.want)
}
}
if _, err := toolsGoSource(Options{ToolsSource: Source{Repository: "http://forge/other/base.git"}}); err == nil ||
!strings.Contains(err.Error(), "--tools-go-source") {
t.Errorf("a base not named mesh-tools, and no Go toolchain given, was not refused naming the flag: %v", err)
}
}
+34 -2
View File
@@ -137,7 +137,26 @@ func registerAndAssign(ctx context.Context, o Options, control controlPlane, mod
// should hold, seals every secret in them and publishes them, and on a first node that is the // should hold, seals every secret in them and publishes them, and on a first node that is the
// slowest thing the mesh does. // slowest thing the mesh does.
func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) { func pushNode(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
said, err := control.within(o.Wait).tell(ctx, "push", o.Node) return pushing(ctx, o, control, false, say)
}
// pushNodeApplied is pushNode that returns only once the node reports it applied exactly what it was
// sent (`push <node> --wait`, novox/hq ADR 0010): for a push whose content the very next step reads
// on the machine. A plain push returns as soon as the declaration is sent, and the node applies it a
// moment later (issue 615).
func pushNodeApplied(ctx context.Context, o Options, control controlPlane, say func(string)) (string, error) {
return pushing(ctx, o, control, true, say)
}
func pushing(ctx context.Context, o Options, control controlPlane, applied bool, say func(string)) (string, error) {
args := []string{"push", o.Node}
patience := o.Wait
if applied {
args = append(args, "--wait", o.Wait.String())
// The controller's own wait, and time to compose and send before it starts.
patience = 2 * o.Wait
}
said, err := control.within(patience).tell(ctx, args...)
if err != nil { if err != nil {
return "", fmt.Errorf( return "", fmt.Errorf(
"%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+ "%w\n\nWhat was registered and assigned is registered and assigned, and this node has "+
@@ -145,7 +164,20 @@ func pushNode(ctx context.Context, o Options, control controlPlane, say func(str
"at all. Fix what it named and run this installer again: it will find the module "+ "at all. Fix what it named and run this installer again: it will find the module "+
"already registered and try the push again", err) "already registered and try the push again", err)
} }
say(" pushed " + o.Node) if applied {
say(" pushed " + o.Node + ", and it applied it")
} else {
say(" pushed " + o.Node)
}
// **Every push may carry a new bus user, and until the adoption the bus hears of it only from here**
// (novox/hq issue 608, ADR 0331 §5, ADR 0333 §4). A module's account is minted into the mesh's
// records and sealed to this node with the push; the bus reads its user list from a file, and until
// the broker step adopts the bus as the nats module nothing but this installer writes that file. The
// list was placed only when the node enrolled, so the build agent's account (step 13) never reached
// the bus and the agent was refused ("Authorization Violation") until the sdk step gave up.
if err := placeUntilAdopted(ctx, control, say); err != nil {
return "", err
}
return strings.TrimSpace(said), nil return strings.TrimSpace(said), nil
} }
+47 -5
View File
@@ -25,20 +25,62 @@ import (
// buildWait bounds one module build. Generous, because the first build compiles a toolchain. // buildWait bounds one module build. Generous, because the first build compiles a toolchain.
const buildWait = 20 * time.Minute const buildWait = 20 * time.Minute
// BuildBase asks the mesh to build the shared base every module with code of its own stands on. // BuildBase asks the mesh to build the shared base every module with code of its own stands on, and
// the toolchain of every language genesis's own modules are written in.
// //
// **First, because until it exists nothing else with code can be built.** Not registered as a // **First, because until it exists nothing else with code can be built.** Not registered as a
// module here: it is never assigned — it runs nowhere — and the build itself records what was // module here: it is never assigned — it runs nowhere — and the build itself records what was
// made, which is all anything downstream reads. // made, which is all anything downstream reads.
//
// **Every toolchain a genesis module compiles in** (novox/hq issue 616). A bundle is compiled in its
// language's base (mesh-controller internal/builder toolchain.go): TypeScript in mesh-tools, Go in
// mesh-tools-go. The store's and the bus's modules carry Go code, so a genesis that built only
// mesh-tools stopped at the store step: "this mesh holds no copy of it. Build mesh-tools-go first".
// The bases are the ones genesisToolchains names, and a test holds them to the languages of every
// module genesis installs, read from the catalogue.
func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error { func BuildBase(ctx context.Context, o Options, control controlPlane, say func(string)) error {
if o.ToolsSource.Repository == "" { if o.ToolsSource.Repository == "" {
return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " + return fmt.Errorf("phase two needs --tools-source: the shared base is built from its " +
"own repository, and an installer told nothing cannot know where that is") "own repository, and an installer told nothing cannot know where that is")
} }
say(" building " + o.ToolsSource.Repository + " at " + refOr(o.ToolsSource.Ref)) goSource, err := toolsGoSource(o)
_, err := control.within(buildWait).tell(ctx, if err != nil {
"build", o.ToolsSource.Repository, "--ref", refOr(o.ToolsSource.Ref), "--wait", "1200s") return err
return err }
for _, src := range []Source{o.ToolsSource, goSource} {
say(" building " + src.Repository + " at " + refOr(src.Ref))
if _, err := control.within(buildWait).tell(ctx,
"build", src.Repository, "--ref", refOr(src.Ref), "--wait", "1200s"); err != nil {
return err
}
}
return nil
}
// genesisToolchains is the base each language's bundles compile in, for the languages genesis's own
// modules are written in — the same pairs as mesh-controller internal/builder toolchain.go — in the
// order BuildBase builds them.
var genesisToolchains = []struct{ Language, Base string }{
{"typescript", "mesh-tools"},
{"go", "mesh-tools-go"},
}
// toolsGoSource is where the Go toolchain is built from: as given, or the repository beside the base's
// named mesh-tools-go, which is how the mesh keeps its repositories side by side.
func toolsGoSource(o Options) (Source, error) {
if o.ToolsGoSource.Repository != "" {
return o.ToolsGoSource, nil
}
repo := strings.TrimSuffix(o.ToolsSource.Repository, "/")
suffix := ""
if strings.HasSuffix(repo, ".git") {
repo, suffix = strings.TrimSuffix(repo, ".git"), ".git"
}
if !strings.HasSuffix(repo, "mesh-tools") {
return Source{}, fmt.Errorf("the Go toolchain is built from mesh-tools-go, and --tools-source (%s) is not a "+
"repository named mesh-tools to find it beside; give --tools-go-source", o.ToolsSource.Repository)
}
return Source{Repository: repo + "-go" + suffix, Ref: o.ToolsGoSource.Ref}, nil
} }
// InstallFromCatalogue builds a catalogue module and installs it on this machine. // InstallFromCatalogue builds a catalogue module and installs it on this machine.
+201 -29
View File
@@ -4,7 +4,9 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/http"
"strings" "strings"
"time"
"github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/declaration"
) )
@@ -82,7 +84,7 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
say); err != nil { say); err != nil {
return err return err
} }
say(" adopted mesh-store — the foundation's store is now the " + module + " module") say(" adopted " + store.Name + " — the foundation's store is now the " + module + " module")
return nil return nil
} }
@@ -91,7 +93,8 @@ func InstallStore(ctx context.Context, o Options, control controlPlane,
// be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker // be in it. InstallFromCatalogue is the same sequence without that moment; the store, the broker
// and the vault each need it or need the shape, and three copies of it drifted. // and the vault each need it or need the shape, and three copies of it drifted.
func installProvider(ctx context.Context, o Options, control controlPlane, module string, func installProvider(ctx context.Context, o Options, control controlPlane, module string,
manifest []byte, buildNote string, beforePush func() error, say func(string)) error { manifest []byte, buildNote string, beforePush func() error, say func(string),
send ...func() error) error {
remote := "/" + module + "-module.json" remote := "/" + module + "-module.json"
if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil { if err := control.carrying(ctx, module+"-module.json", manifest, remote); err != nil {
@@ -130,6 +133,11 @@ func installProvider(ctx context.Context, o Options, control controlPlane, modul
return err return err
} }
} }
// A module whose first send is not a plain push says how it is sent: the bus, whose replacement is a
// planned step (ADR 0236), is sent by that step (adoptTheBus).
if len(send) > 0 && send[0] != nil {
return send[0]()
}
_, err := pushNode(ctx, o, control, say) _, err := pushNode(ctx, o, control, say)
return err return err
} }
@@ -165,13 +173,22 @@ const BusModule = "nats"
// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the // (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the
// bridge to here and is not run again. // bridge to here and is not run again.
// //
// **What is compared is the container's name, not its image.** The store's module pins the upstream // **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's // name, the same streams directory, configuration directory, user list and certificate. That is checked
// snapshot tool), so no pinned reference can be equal to the foundation's. A module whose server is // here against the module's own manifest before anything is registered, and a difference is refused
// not the foundation's container by name would raise a second bus beside the first, and is refused // naming each one — under another name or on other paths the module would raise a second bus beside the
// by name before anything is registered. // first, or an empty one in its place.
//
// **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot
// tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on
// the module's image, on the same streams, users and certificate. The mesh replaces its bus only as a
// planned step (ADR 0236), so the module is sent by that step: its streams snapshotted first, the restart
// said, and the installer waits until the container runs the module's image, the bus answers, and the
// controller ends the step done (bus_adoption.go), so nothing after this step dials a bus that is restarting.
func InstallBroker(ctx context.Context, o Options, control controlPlane, func InstallBroker(ctx context.Context, o Options, control controlPlane,
foundation *declaration.Declaration, say func(string)) error { foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error),
say func(string)) error {
const module = BusModule const module = BusModule
manifest, err := readManifest(o.Catalogue, module) manifest, err := readManifest(o.Catalogue, module)
@@ -182,12 +199,18 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane,
if err != nil { if err != nil {
return err return err
} }
if err := busServerIsTheFoundations(manifest, bus, module); err != nil { if err := busLayoutIsTheModules(manifest, foundation, module); err != nil {
return err return err
} }
say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept") say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept")
if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)", if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
nil, say); err != nil { nil, say, func() error { return adoptTheBus(ctx, o, control, bus, say) }); err != nil {
return err
}
if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil {
return err
}
if err := waitForTheBusStep(ctx, o, control, say); err != nil {
return err return err
} }
say(" adopted " + bus.Name + " — the foundation's bus is now the " + module + say(" adopted " + bus.Name + " — the foundation's bus is now the " + module +
@@ -195,41 +218,190 @@ func InstallBroker(ctx context.Context, o Options, control controlPlane,
return nil return nil
} }
// busServerIsTheFoundations checks that the bus module's server container is the one the // busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own
// foundation raised, by name, so the applier takes it over rather than raising a second bus. // loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only
func busServerIsTheFoundations(manifest []byte, bus *declaration.Container, module string) error { // once the server is up and JetStream has its streams back.
const busMonitor = "http://127.0.0.1:8222/healthz"
// runningImage is the image the named container was created from, or "" when there is none.
func runningImage(ctx context.Context, control controlPlane, name string) string {
asking, cancel := context.WithTimeout(ctx, control.timeout)
defer cancel()
out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name)
if err != nil {
return ""
}
return strings.TrimSpace(out)
}
// waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised
// it from — the module's, after the applier's one recreate — and the bus answers on its monitor.
//
// Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is
// refused, naming what it runs: the push was sent and the node has not applied it, and every step after
// this one reaches the mesh through that bus.
func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container,
fetch func(context.Context, string) (int, string, error), say func(string)) error {
deadline := time.Now().Add(o.Wait)
var running, said string
for {
running = runningImage(ctx, control, bus.Name)
if running != "" && running != bus.Image {
code, body, err := fetch(ctx, busMonitor)
if err == nil && code == http.StatusOK {
say(" bus answers " + bus.Name + " on " + running)
return nil
}
said = strings.TrimSpace(body)
if err != nil {
said = err.Error()
}
}
if !time.Now().Before(deadline) {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(answerEvery):
}
}
switch {
case running == "":
return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+
"after this step could reach the mesh. The node-engine's journal says why it did not raise it",
bus.Name, BusModule)
case running == bus.Image:
return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+
"has not applied it. Its streams are untouched; run this installer again once the node-engine "+
"applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule)
}
return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s",
bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said))
}
// BusUsersID is the bundle's resource for the bus's first user list.
const BusUsersID = "bus-accounts"
// busLayout is where a bus lives: its container, the machine's directories it mounts as its streams,
// its configuration and its certificate, and the file its user list is.
type busLayout struct {
Container, Streams, Conf, Certificate, Users string
}
// Where the server reads each, inside its container: the same in the bundle and the module, since both
// run the same server with the same configuration.
const (
busStreamsInside = "/data"
busConfInside = "/etc/nats"
busCertificateInside = "/tls"
)
// foundationBusLayout reads the layout of the bus the bundle raises.
func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) {
bus, err := brokerIn(foundation)
if err != nil {
return busLayout{}, err
}
at := mountedFrom(bus.Volumes)
l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
Certificate: at[busCertificateInside]}
for _, r := range foundation.Resources {
if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID {
l.Users = f.Path
}
}
return l, nil
}
// moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of
// that container with every `${access:<id>}` resolved to the access's path, and its `bus-users`.
func moduleBusLayout(manifest []byte, module string) (busLayout, error) {
var m struct { var m struct {
BusUsers string `json:"bus-users"`
Resources []struct { Resources []struct {
Type string `json:"type"` ID string `json:"id"`
Name string `json:"name"` Type string `json:"type"`
Name string `json:"name"`
Volumes []string `json:"volumes"`
} `json:"resources"` } `json:"resources"`
Accesses []struct {
ID string `json:"id"`
Path string `json:"path"`
} `json:"accesses"`
} }
if err := json.Unmarshal(manifest, &m); err != nil { if err := json.Unmarshal(manifest, &m); err != nil {
return fmt.Errorf("the %s module's manifest is not readable: %w", module, err) return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
} }
var names []string var containers []int
for _, r := range m.Resources { for i, r := range m.Resources {
if r.Type != "container" { if r.Type == "container" {
continue containers = append(containers, i)
} }
if r.Name == bus.Name { }
return nil if len(containers) != 1 {
var names []string
for _, i := range containers {
names = append(names, m.Resources[i].Name)
} }
names = append(names, r.Name) return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+
"declares one container", module, describeNames(names))
}
server := m.Resources[containers[0]]
resolved := make([]string, 0, len(server.Volumes))
for _, v := range server.Volumes {
for _, a := range m.Accesses {
v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path)
}
resolved = append(resolved, v)
}
at := mountedFrom(resolved)
return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
Certificate: at[busCertificateInside], Users: m.BusUsers}, nil
}
// busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by
// field, so the applier takes it over rather than raising a second bus beside it or an empty one in its
// place (novox/hq ADR 0333 §1). Every difference is named, with both sides.
func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error {
theirs, err := moduleBusLayout(manifest, module)
if err != nil {
return err
}
ours, err := foundationBusLayout(foundation)
if err != nil {
return err
}
var differ []string
for _, f := range []struct{ what, module, bundle string }{
{"the server container", theirs.Container, ours.Container},
{"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams},
{"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf},
{"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate},
{"its user list", theirs.Users, ours.Users},
} {
if f.module == "" || f.module != f.bundle {
differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what,
f.module, f.bundle))
}
}
if len(differ) == 0 {
return nil
} }
return fmt.Errorf( return fmt.Errorf(
"the %s module's server is %s, and the foundation's bus is the container %q.\n"+ "the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+
"The module adopts the foundation's bus in place, so its server has to carry the name the "+ "The module adopts the foundation's bus in place, so the bundle raises it with the module's "+
"foundation raised; under another name it would raise a second bus beside the first, and "+ "name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+
"the mesh would run two (novox/hq ADR 0331 §1). Nothing has been registered", "one on other paths (novox/hq ADR 0333 §1). Nothing has been registered",
module, describeNames(names), bus.Name) module, strings.Join(differ, "\n - "))
} }
func describeNames(names []string) string { func describeNames(names []string) string {
if len(names) == 0 { if len(names) == 0 {
return "no container at all" return "no container at all"
} }
return "the container " + strings.Join(names, ", ") return "the containers " + strings.Join(names, ", ")
} }
// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to // InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
+370 -16
View File
@@ -2,10 +2,16 @@ package bootstrap
import ( import (
"context" "context"
"encoding/json"
"fmt"
"net/http"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
) )
// aCatalogue is a catalogue checkout holding these modules' manifests. // aCatalogue is a catalogue checkout holding these modules' manifests.
@@ -24,32 +30,74 @@ func aCatalogue(t *testing.T, manifests map[string]string) string {
return dir return dir
} }
// aBusModule is the nats module's manifest as far as adoption reads it: a server container by name. // aBusModule is the nats module's manifest as far as adoption reads it: its server container, the
func aBusModule(server string) string { // directories it mounts, its certificate through an access, and its user list — the shape of the
// catalogue's modules/nats/module.json.
func aBusModule(l busLayout) string {
return `{"module": "nats", "version": "1", return `{"module": "nats", "version": "1",
"claims": [{"name": "mesh-broker", "scope": "mesh"}], "claims": [{"name": "mesh-broker", "scope": "mesh"}],
"bus-users": "/var/lib/nats-module/conf/accounts.conf", "bus-users": "` + l.Users + `",
"resources": [{"id": "server", "type": "container", "name": "` + server + `", "ports": ["4222:4222"], "artifact": "server"}], "resources": [
{"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"],
"volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}],
"accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}],
"build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}` "build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}`
} }
func busStep(t *testing.T, catalogue string) (*asked, error) { // theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats).
var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats",
Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls",
Users: "/var/lib/nats-module/conf/accounts.conf"}
// theModulesImage is what the bus runs once the applier recreated it on the module's built image.
const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333"
// aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the
// module's image, and its monitor says ok.
func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) {
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
"inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n",
})}
return runtime, func(_ context.Context, url string) (int, string, error) {
if url != busMonitor {
return 0, "", fmt.Errorf("asked %s", url)
}
return http.StatusOK, `{"status":"ok"}`, nil
}
}
func busStepWith(t *testing.T, o Options, runtime *asked,
fetch func(context.Context, string) (int, string, error)) ([]string, error) {
t.Helper() t.Helper()
aGenesisBusOnDisk(t)
rewritten, err := Rewrite(theRealBundle(t), held) rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
runtime := &asked{answer: aMeshThatAgrees(nil)} control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: 0} var said []string
err = InstallBroker(context.Background(), installing(t, catalogue), control, rewritten.Declaration, func(string) {}) err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch,
func(line string) { said = append(said, line) })
return said, err
}
func busStep(t *testing.T, catalogue string) (*asked, error) {
t.Helper()
runtime, fetch := aBusThatAdopts()
o := installing(t, catalogue)
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
_, err := busStepWith(t, o, runtime, fetch)
return runtime, err return runtime, err
} }
// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module // The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module
// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it. // the mesh goes on managing, registered from the catalogue — never lavinmq, which left it.
func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) { func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker")}) catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
runtime, _ := busStep(t, catalogue) runtime, err := busStep(t, catalogue)
if err != nil {
t.Fatal(err)
}
if !runtime.ran("module add /nats-module.json") { if !runtime.ran("module add /nats-module.json") {
t.Fatalf("the nats module was not registered: %v", runtime.commands) t.Fatalf("the nats module was not registered: %v", runtime.commands)
} }
@@ -74,15 +122,23 @@ func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) {
} }
} }
// A bus module whose server is not the foundation's container by name would raise a second bus // A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus
// beside the first; it is refused naming both, before anything is registered. // beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every
func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) { // difference with both sides, before anything is registered. The layout is the one the bundle had
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule("mesh-broker-nats")}) // before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590).
func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) {
before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf",
Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"}
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)})
runtime, err := busStep(t, catalogue) runtime, err := busStep(t, catalogue)
if err == nil { if err == nil {
t.Fatal("a bus module with another server was adopted") t.Fatal("a bus module kept elsewhere was adopted")
} }
for _, named := range []string{"mesh-broker-nats", `"mesh-broker"`} { for _, named := range []string{
`"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`,
`"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`,
`"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`,
} {
if !strings.Contains(err.Error(), named) { if !strings.Contains(err.Error(), named) {
t.Errorf("the refusal does not name %s: %v", named, err) t.Errorf("the refusal does not name %s: %v", named, err)
} }
@@ -91,3 +147,301 @@ func TestABusModuleWhoseServerIsNotTheFoundationsIsRefused(t *testing.T) {
t.Errorf("something was registered: %v", runtime.commands) t.Errorf("something was registered: %v", runtime.commands)
} }
} }
// One field moved is refused as surely as all of them: a module whose streams are elsewhere would be
// raised empty on the bus's own name.
func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) {
moved := theModulesLayout
moved.Streams = "/srv/nats"
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)})
runtime, err := busStep(t, catalogue)
if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") {
t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err)
}
if strings.Contains(err.Error(), "certificate") {
t.Errorf("the refusal names what agrees: %v", err)
}
if runtime.ran("module add") {
t.Errorf("something was registered: %v", runtime.commands)
}
}
// The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step
// gives it the setting that maps that to the port the bundle published, 5671 unless given — before the
// push, so the first declaration already names it.
func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) {
for _, c := range []struct {
given int
want string
}{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
runtime, fetch := aBusThatAdopts()
o := installing(t, catalogue)
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
o.Ports = FoundationPorts{Bus: c.given}
said, err := busStepWith(t, o, runtime, fetch)
if err != nil {
t.Fatal(err)
}
set, pushed := -1, -1
for i, command := range runtime.commands {
if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 {
set = i
}
if strings.Contains(command, "push anchor") && pushed < 0 {
pushed = i
}
}
if set < 0 || pushed < 0 || set > pushed {
t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands)
}
if !strings.Contains(strings.Join(said, "\n"), "settings nats on anchor: "+c.want) {
t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n"))
}
}
}
// The one restart is said before it happens, and the step goes on only once the bus runs the module's
// image and answers (novox/hq ADR 0333 §3).
func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
// The push is sent; the node takes a few passes to recreate the bus, and the server a few more to
// answer.
pushed, inspected, monitored := false, 0, 0
runtime := &asked{}
agrees := aMeshThatAgrees(nil)
runtime.answer = func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case strings.Contains(joined, "bus upgrade --why"):
pushed = true
case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
if !pushed {
return bundleImage + "\n", nil
}
inspected++
if inspected < 3 {
return bundleImage + "\n", nil
}
return theModulesImage + "\n", nil
}
return agrees(name, args)
}
fetch := func(context.Context, string) (int, string, error) {
monitored++
if monitored < 3 {
return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil
}
return http.StatusOK, `{"status":"ok"}`, nil
}
o := installing(t, catalogue)
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
o.Wait = 10 * time.Second
said, err := busStepWith(t, o, runtime, fetch)
if err != nil {
t.Fatal(err)
}
text := strings.Join(said, "\n")
restart := strings.Index(text, "restarting mesh-broker-nats — once")
answers := strings.Index(text, "bus answers mesh-broker-nats on "+theModulesImage)
adopted := strings.Index(text, "adopted mesh-broker-nats")
if restart < 0 || answers < restart || adopted < answers {
t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text)
}
if inspected < 3 || monitored < 3 {
t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+
"times, its monitor %d)", inspected, monitored)
}
}
// A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after
// this one reaches the mesh through that bus, and the push was not applied.
func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) {
previous := answerEvery
answerEvery = time.Millisecond
defer func() { answerEvery = previous }()
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
"inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n",
})}
fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil }
o := installing(t, catalogue)
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
o.Wait = 20 * time.Millisecond
_, err = busStepWith(t, o, runtime, fetch)
if err == nil {
t.Fatal("a bus the node never recreated was taken as adopted")
}
if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") {
t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err)
}
}
// A re-run after the adoption finds the bus on the module's image already, and says so rather than
// announcing a restart that does not happen.
func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) {
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
runtime, fetch := aBusThatAdopts()
o := installing(t, catalogue)
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
said, err := busStepWith(t, o, runtime, fetch)
if err != nil {
t.Fatal(err)
}
text := strings.Join(said, "\n")
if strings.Contains(text, "restarting") || !strings.Contains(text, "already adopted mesh-broker-nats") {
t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text)
}
}
// theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333,
// "How it is checked"): in a merge check the clone the build seat puts beside this repository
// (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue
// beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check
// never skips it.
func theCataloguesBusModule(t *testing.T) []byte {
t.Helper()
const inside = "mesh-catalog/modules/nats/module.json"
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
raw, err := os.ReadFile(filepath.Join(root, inside))
if err != nil {
t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+
"cannot be held to the module's: %v", root, err)
}
t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside))
return raw
}
beside := filepath.Join("..", "..", "..", inside)
raw, err := os.ReadFile(beside)
if err != nil {
t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+
"bus is not held to the nats module's manifest here; the merge check holds it", beside)
}
t.Logf("judged against the catalogue checked out at %s", beside)
return raw
}
// **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's
// manifest: the bus's container, its streams, its configuration, its certificate and its user list are
// where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised
// `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590).
func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) {
manifest := theCataloguesBusModule(t)
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil {
t.Fatal(err)
}
// And the port the installer's setting maps is the one the module declares (ADR 0333 §2).
var m struct {
Resources []struct {
Type string `json:"type"`
Ports []string `json:"ports"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
t.Fatal(err)
}
declared := false
for _, r := range m.Resources {
for _, p := range r.Ports {
if p == fmt.Sprintf("%d:4222", busModulePort) {
declared = true
}
}
}
if !declared {
t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+
"to the node's bus port", busModulePort)
}
}
// The bundle as the installer reads it agrees with the module's layout as this repository states it,
// and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the
// bus reads, and makes the container that reads it re-read it.
func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) {
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
ours, err := foundationBusLayout(rewritten.Declaration)
if err != nil {
t.Fatal(err)
}
if ours != theModulesLayout {
t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout)
}
if ours.Users != busAccounts || ours.Container != busContainer {
t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s",
busAccounts, busContainer, ours.Users, ours.Container)
}
if filepath.Dir(ours.Users) != ours.Conf {
t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf)
}
}
// The bundle's bus configuration and user list are seeds (create-once): an installer run again after
// the adoption applies the bundle again, and must not put the bundle's one user back over the users the
// module writes there (ADR 0333 §4).
func TestTheBundlesBusFilesAreSeeds(t *testing.T) {
rewritten, err := Rewrite(theRealBundle(t), held)
if err != nil {
t.Fatal(err)
}
seeds := map[string]bool{}
for _, r := range rewritten.Declaration.Resources {
if f, ok := r.(*declaration.File); ok && f.CreateOnce {
seeds[f.Path] = true
}
}
for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} {
if !seeds[path] {
t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path)
}
}
}
// aGenesisBusOnDisk puts what the adoption reads on the machine in a test's own directory: the bus's
// certificate, the controller's bus credential, and where the snapshot goes. The step's wait is short.
func aGenesisBusOnDisk(t *testing.T) {
t.Helper()
dir := t.TempDir()
cert := filepath.Join(dir, "tls.crt")
if err := os.WriteFile(cert, []byte(aCertificatePEM), 0o644); err != nil {
t.Fatal(err)
}
secret := filepath.Join(dir, "bus-controller.secret")
if err := os.WriteFile(secret, []byte("BUS-PW"), 0o600); err != nil {
t.Fatal(err)
}
was := []string{busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret}
wasWithin := busStepWithin
busSnapshotFile, busSnapshotCredential = filepath.Join(dir, "bus.tar"), filepath.Join(dir, "bus.credential")
busCertificateFile, busControllerSecret = cert, secret
busStepWithin = 50 * time.Millisecond
t.Cleanup(func() {
busSnapshotFile, busSnapshotCredential, busCertificateFile, busControllerSecret = was[0], was[1], was[2], was[3]
busStepWithin = wasWithin
})
}
// aCertificatePEM is a PEM block standing for the bus's certificate; its pin is the sha256 of these bytes.
const aCertificatePEM = "-----BEGIN CERTIFICATE-----\nAAECAwQFBgcICQ==\n-----END CERTIFICATE-----\n"
+67 -24
View File
@@ -2,6 +2,7 @@ package bootstrap
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"os" "os"
@@ -24,7 +25,7 @@ import (
const ( const (
// foundationStore is the foundation's postgres container — the mesh's own memory, raised from the // foundationStore is the foundation's postgres container — the mesh's own memory, raised from the
// bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051). // bundle. gitea's bootstrap database lives here too, so a mesh runs one postgres (issue 051).
foundationStore = "mesh-store" foundationStore = "postgres"
// giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module — // giteaBootstrap is the gitea server raised directly at genesis, before gitea is a module —
// under the name the gitea MODULE declares for its container, so the module finds it and holds // under the name the gitea MODULE declares for its container, so the module finds it and holds
// it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090). // it rather than raising a second forge beside it (novox/hq ADR 0163, rule 7; issue 090).
@@ -49,8 +50,8 @@ const (
packagesTeam = "packages" packagesTeam = "packages"
// giteaAdminUser is the admin the bootstrap creates and the provisioner later authenticates as. // giteaAdminUser is the admin the bootstrap creates and the provisioner later authenticates as.
giteaAdminUser = "mesh-admin" giteaAdminUser = "mesh-admin"
// builderGiteaUser is the gitea account the builder publishes and pulls with at genesis. It is // builderGiteaUser is the gitea account the build agent publishes and pulls with at genesis: the
// the `as` the builder's static package binding names. // `as` of the genesis document (novox/hq ADR 0336 §3). Retiring it is issue 602.
builderGiteaUser = "mesh-builder" builderGiteaUser = "mesh-builder"
// giteaDBRole/giteaDBName is gitea's own database in the foundation store. // giteaDBRole/giteaDBName is gitea's own database in the foundation store.
giteaDBRole = "mesh_gitea" giteaDBRole = "mesh_gitea"
@@ -133,8 +134,8 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro
return err return err
} }
say(" delivering the builder its registry credential") say(" sealing the build agent the genesis document for the forge it raised")
if err := deliverBuilderNpm(ctx, o, control, builderPassword, say); err != nil { if err := deliverGenesisRegistry(ctx, o, control, genesisRegistry(ports, builderPassword), say); err != nil {
return err return err
} }
@@ -362,27 +363,69 @@ func packagePasswords() (db, admin, builder string, err error) {
return db, admin, builder, nil return db, admin, builder, nil
} }
// deliverBuilderNpm seals the builder's registry password to this node as its `npm-password` // GenesisRegistrySecret is the build agent's own-secret the genesis document is sealed as (novox/hq
// own-secret, the same way the control plane's store connections are delivered — carry the value in, // ADR 0336 §3). The build-agent manifest declares it, issued outside the mesh: only this installer
// `secret accept`, and the next push writes it sealed where the builder reads it. // gives one, on genesis's node.
func deliverBuilderNpm(ctx context.Context, o Options, control controlPlane, password string, const GenesisRegistrySecret = "genesis-package-registry"
say func(string)) error {
at := "/accepting-npm-password" // GenesisRegistry is the genesis document (novox/hq ADR 0336 §3): the forge this step raised, the port
if err := control.carryingSecret(ctx, "mesh-accepting-npm-password", []byte(password), at); err != nil { // this node gave the package registry, its npm path, and the account this step made with its password.
return err // mesh-controller's cmd/mesh-builder reads it under the same field names while the build agent's
} // npm-package-registry requirement is unbound, and prefers the binding once the gitea module is
if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, "npm-password", "--from", at); err != nil { // assigned and it binds.
return err type GenesisRegistry struct {
} Scheme string `json:"scheme"`
// Push so the sealed secret reaches the builder, which restarts on it and comes back credentialed. At string `json:"at"`
if _, err := control.tell(ctx, "push", o.Node); err != nil { Port int `json:"port"`
return err NpmPath string `json:"npm-path"`
} As string `json:"as"`
return nil Password string `json:"password"`
} }
// PublishTheSDK dispatches a build of the SDK to the builder. The builder has its registry // genesisRegistry is the document for the forge raised here: on the machine's own loopback, where it
// credential by now, so the build's `npm publish` authenticates; the artifact is a `package`, built // answers on the host network and where its ROOT_URL says it is (raiseGiteaServer), on the port this
// node gave the package registry.
func genesisRegistry(ports FoundationPorts, password string) GenesisRegistry {
return GenesisRegistry{Scheme: "http", At: "127.0.0.1", Port: ports.Packages,
NpmPath: "/api/packages/" + packagesOrg + "/npm/", As: builderGiteaUser, Password: password}
}
// deliverGenesisRegistry seals the genesis document to the build agent on this node as its
// `genesis-package-registry` own-secret (novox/hq ADR 0336 §3), the same way the control plane's store
// connections are delivered — carry the value in, `secret accept`, and the next push writes it sealed
// where the build agent reads it.
//
// **The build agent's one way to the forge until the registry binds.** Its npm-package-registry
// requirement is optional (ADR 0336 §1), and nothing provides it until the gitea module is assigned
// after the base, so the sdk step (15) publishes and the base step (16) resolves with the account in
// this document. It replaces the retired builder's `npm-password` own-secret and the `serves.port`
// setting its carried binding took, neither of which the build agent declares.
func deliverGenesisRegistry(ctx context.Context, o Options, control controlPlane, doc GenesisRegistry,
say func(string)) error {
raw, err := json.Marshal(doc)
if err != nil {
return err
}
at := "/accepting-" + GenesisRegistrySecret
if err := control.carryingSecret(ctx, "mesh-accepting-"+GenesisRegistrySecret, raw, at); err != nil {
return err
}
if _, err := control.tell(ctx, "secret", "accept", o.Node, BuilderModule, GenesisRegistrySecret, "--from", at); err != nil {
return err
}
// Push so the sealed document reaches the build agent, which reads it at its next build — and,
// like every push before the adoption, the bus hears the users it carries (issue 608).
//
// **Waited for until the node applied it** (issue 615). The next step asks the build agent to
// publish the SDK, and the agent reads the document from the file the node writes when it applies
// this push. A push that returned on sending let the ask race the apply: one genesis found the
// document in time, the next was refused, "neither its binding nor the genesis document".
_, err = pushNodeApplied(ctx, o, control, say)
return err
}
// PublishTheSDK dispatches a build of the SDK to the build agent. The agent has the genesis document by
// now (novox/hq ADR 0336 §7), so the build's `npm publish` authenticates; the artifact is a `package`, built
// on a public base, so this needs no toolchain — which is the whole point of doing it before the base. // on a public base, so this needs no toolchain — which is the whole point of doing it before the base.
func PublishTheSDK(ctx context.Context, o Options, control controlPlane, say func(string)) error { func PublishTheSDK(ctx context.Context, o Options, control controlPlane, say func(string)) error {
if o.SDKSource.Repository == "" { if o.SDKSource.Repository == "" {
+2 -25
View File
@@ -117,24 +117,6 @@ const busModulePort = 4222
// PortsSetting is the controller's settings key for a module's given ports. // PortsSetting is the controller's settings key for a module's given ports.
const PortsSetting = "ports" const PortsSetting = "ports"
// ServesSetting is the settings key that carries what a provider serves, inside the one binding a
// module carries rather than resolves — the builder's package binding (novox/hq 04-ISSUES/085).
//
// **A port given at genesis has to be a setting wherever it is read, or something puts it back.**
// The builder reaches the package registry through a binding written in its own manifest, because
// at genesis no module yet provides `package-registry` and so there is nothing for the controller
// to resolve it from. That binding used to be rewritten, as text, when the installer registered the
// builder — which a later registration from the catalogue undid silently. Set as a node's setting
// instead, it is held by the controller rather than by the manifest, so re-registering the builder
// leaves it where it was.
//
// **A settings layer is the module's, not one resource's.** The controller lays it over every file
// of that module which merges as JSON, so `serves` lands in the package binding only because that
// binding is the builder's one mergeable file. A second mergeable file added to the builder would
// be given a `serves` key too, meaning nothing to whatever reads it. Worth knowing before adding
// one.
const ServesSetting = "serves"
// setFoundationSettings tells the controller the ports this node gave a foundation module — and, // setFoundationSettings tells the controller the ports this node gave a foundation module — and,
// on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it // on an adopted node, that the registry is reached from anywhere, as a node pulls from it before it
// has a private-network address (novox/hq ADR 0100). Done after the module is registered and before // has a private-network address (novox/hq ADR 0100). Done after the module is registered and before
@@ -148,13 +130,8 @@ func setFoundationSettings(ctx context.Context, o Options, control controlPlane,
if o.Adopted && module == RegistryModule { if o.Adopted && module == RegistryModule {
values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"} values["expose"] = map[string]string{strconv.Itoa(DefaultPorts().Registry): "anywhere"}
} }
if packages := o.Ports.orDefaults().Packages; module == BuilderModule && // No `serves` setting for the build agent: the port given for the package registry reaches it in
packages != DefaultPorts().Packages { // the genesis document (novox/hq ADR 0336 §3), and from the binding once the registry binds.
// The one binding nothing resolves: the builder dials the forge by a number it carries.
// The rest of the binding — the provision, the forge it is from, the account it presents —
// is the manifest's and stays there.
values[ServesSetting] = map[string]int{"port": packages}
}
if len(values) == 0 { if len(values) == 0 {
return nil return nil
} }
+8 -2
View File
@@ -68,11 +68,11 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
if got.Places == 0 { if got.Places == 0 {
t.Fatal("nothing was rewritten") t.Fatal("nothing was rewritten")
} }
storeC := containerNamed(r.Declaration, "mesh-store") storeC := containerNamed(r.Declaration, "postgres")
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" { if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
t.Errorf("the store publishes %v", storeC.Ports) t.Errorf("the store publishes %v", storeC.Ports)
} }
bus := containerNamed(r.Declaration, "mesh-broker") bus := containerNamed(r.Declaration, "mesh-broker-nats")
if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" { if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" {
t.Errorf("the bus publishes %v", bus.Ports) t.Errorf("the bus publishes %v", bus.Ports)
} }
@@ -239,6 +239,12 @@ func (c *controlRecorder) run(_ context.Context, name string, args ...string) (s
} }
if name == "docker" && args[0] == "exec" { if name == "docker" && args[0] == "exec" {
c.told = append(c.told, strings.Join(args[3:], " ")) c.told = append(c.told, strings.Join(args[3:], " "))
if strings.Join(args[3:], " ") == "broker accounts" {
return "accounts {}\n", nil // the composed user list, placed after every push (issue 608)
}
}
if name == "sh" {
c.told = append(c.told, "placed the bus's users")
} }
return "", nil return "", nil
} }
@@ -0,0 +1,134 @@
package bootstrap
import (
"context"
"strings"
"testing"
"time"
)
// Defends novox/hq issue 608 (ADR 0331 §5, ADR 0333 §4): until the bus is adopted as the nats module,
// every push genesis makes is followed by the installer placing the composed user list where the bus
// reads it, and making the bus re-read it. The build agent's account was minted at step 13 and sealed
// with that push, and the list was placed only at enrolment, so the bus refused the agent
// ("Authorization Violation") and genesis stopped at step 15.
// placing answers the controller as a mesh whose bus is or is not the nats module yet, and records
// what was run, in order.
func placing(adopted bool) (*asked, controlPlane) {
listed := "build-agent 1\n"
if adopted {
listed += "nats 1\n"
}
runtime := &asked{}
runtime.answer = func(name string, args []string) (string, error) {
joined := strings.Join(args, " ")
switch {
case name == "sh":
return "", nil
case strings.HasSuffix(joined, "module list"):
return listed, nil
case strings.HasSuffix(joined, "broker accounts"):
return "accounts {\n MESH { users = [ { user: \"anchor.build-agent\" } ] }\n}\n", nil
}
return "", nil
}
return runtime, controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
}
// placedAfterThePush is whether the bus's users were placed — written at the module's bus-users path,
// the bus told to re-read them — after the push, and how often.
func placedAfterThePush(t *testing.T, commands []string) int {
t.Helper()
pushed, placed := -1, 0
for i, c := range commands {
if strings.Contains(c, "/mesh-controller push anchor") && pushed < 0 {
pushed = i
}
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, busAccounts) &&
strings.Contains(c, "kill -s HUP "+busContainer) {
if pushed < 0 || i < pushed {
t.Fatalf("the bus's users were placed before the push that carries the account: %v", commands)
}
placed++
}
}
if pushed < 0 {
t.Fatalf("nothing was pushed: %v", commands)
}
return placed
}
func TestTheBusHearsTheBuildAgentsAccountAfterItsPush(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
runtime, control := placing(false)
o := Options{Node: "anchor", Wait: time.Second}
if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
if n := placedAfterThePush(t, runtime.commands); n != 1 {
t.Fatalf("the bus's users were placed %d time(s) after the build agent's push; once: %v", n, runtime.commands)
}
}
func TestTheGenesisDocumentsPushIsHeardByTheBusToo(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
runtime, control := placing(false)
o := Options{Node: "anchor", Wait: time.Second}
if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil {
t.Fatal(err)
}
if n := placedAfterThePush(t, runtime.commands); n != 1 {
t.Fatalf("the bus's users were placed %d time(s) after the genesis document's push; once: %v", n, runtime.commands)
}
}
// From the adoption on the nats module writes the list, and the installer does not (ADR 0333 §4).
func TestOnceTheBusIsTheNatsModulesThePushPlacesNothing(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
runtime, control := placing(true)
o := Options{Node: "anchor", Wait: time.Second}
if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
if n := placedAfterThePush(t, runtime.commands); n != 0 {
t.Fatalf("the installer placed the bus's users over the nats module's: %v", runtime.commands)
}
}
// Defends novox/hq issue 615: the packages step's push is waited for until the node applied it, because
// the sdk step right after asks the build agent for a build that reads the genesis document from the
// file that apply writes. A push that returned on sending let the ask race the apply.
func TestThePackagesStepWaitsUntilTheNodeAppliedTheGenesisDocument(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
runtime, control := placing(false)
o := Options{Node: "anchor", Wait: 3 * time.Minute}
if err := deliverGenesisRegistry(context.Background(), o, control, genesisRegistry(DefaultPorts(), "pw"), quietly); err != nil {
t.Fatal(err)
}
for _, c := range runtime.commands {
if strings.Contains(c, "/mesh-controller push anchor") {
if !strings.HasSuffix(c, "push anchor --wait 3m0s") {
t.Fatalf("the genesis document's push does not wait for the node to apply it: %s", c)
}
return
}
}
t.Fatalf("the genesis document was never pushed: %v", runtime.commands)
}
// Every other push of genesis still returns on sending: only a push whose content the next step reads on
// the machine waits.
func TestAnOrdinaryPushDoesNotWait(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
runtime, control := placing(false)
o := Options{Node: "anchor", Wait: time.Second}
if _, err := installModule(context.Background(), o, control, BuilderModule, []byte(`{}`), quietly); err != nil {
t.Fatal(err)
}
for _, c := range runtime.commands {
if strings.Contains(c, "push anchor --wait") {
t.Fatalf("an ordinary push waits: %s", c)
}
}
}
+12
View File
@@ -58,8 +58,20 @@ func aMeshThatAgrees(answers map[string]string) func(string, []string) (string,
} }
} }
switch { switch {
case name == "sh" && len(args) == 2 && strings.Contains(args[1], "accounts.conf"):
return "", nil // the bus's users placed after a push (issue 608)
case name == "sh" && len(args) == 2 && strings.Contains(args[1], "mesh-nats-snapshot"):
return "", nil // the streams snapshotted before the bus's planned step (issue 590)
case name == "docker" && strings.HasSuffix(joined, "/mesh-controller bus"):
return "the bus is nats, built 53e51864, on anchor\nlast step 1: not known → 53e51864 on anchor by installer (" +
adoptionWhy + "): done at 2026-10-13 11:40; snapshot " + busSnapshotFile + "\n", nil
case name == "docker" && strings.Contains(joined, "/mesh-controller builds nats -n 1"):
return "nats built 53e51864 anchor\n image artifact-store://nats/server@sha256:" +
strings.Repeat("4", 64) + "\n bundle artifact-store://nats/tools/blobs/sha256:" + strings.Repeat("5", 64) + "\n", nil
case name != "docker": case name != "docker":
return "", fmt.Errorf("unexpected program %q", name) return "", fmt.Errorf("unexpected program %q", name)
case strings.HasSuffix(joined, "broker accounts"):
return "accounts {}\n", nil
case args[0] == "cp": case args[0] == "cp":
return "", nil return "", nil
case args[0] == "container": case args[0] == "container":
+13 -7
View File
@@ -47,6 +47,9 @@ const (
// StoreSuperuserFile is where the store's superuser password lives on the machine — the // StoreSuperuserFile is where the store's superuser password lives on the machine — the
// postgres module's own-secret path, so genesis and adoption write the same file. // postgres module's own-secret path, so genesis and adoption write the same file.
StoreSuperuserFile = "/var/lib/postgres/superuser.secret" StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
// StoreDataDir is where the store keeps its data — the postgres module's data directory, so
// genesis raises the store where the module adopts it (novox/hq ADR 0338 §1).
StoreDataDir = "/var/lib/mesh-store"
// BusControllerFile is where the controller's password on the bus lives on the machine. Not a // BusControllerFile is where the controller's password on the bus lives on the machine. Not a
// module's path: no module reads it — the permanent controller is handed the address the // module's path: no module reads it — the permanent controller is handed the address the
// temporary one was given — and genesis keeps it only so a second run puts the same account // temporary one was given — and genesis keeps it only so a second run puts the same account
@@ -62,7 +65,7 @@ const (
// What the template says, matched exactly. A template that says something else is a template // What the template says, matched exactly. A template that says something else is a template
// this installer does not know how to make safe, and it says so rather than guessing. // this installer does not know how to make safe, and it says so rather than guessing.
templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"` templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]` templateStoreVolumes = `"volumes": ["` + StoreDataDir + `:/var/lib/postgresql/data"]`
templateStoreURL = "postgres:bootstrap@" templateStoreURL = "postgres:bootstrap@"
// BusURLPlaceholder is the controller's account in its bus address, as the template names it. // BusURLPlaceholder is the controller's account in its bus address, as the template names it.
@@ -265,8 +268,8 @@ func Masking(say func(string), c RootCredentials) func(string) {
// RefuseExistingServers stops a run that would put a made credential in front of a store raised // RefuseExistingServers stops a run that would put a made credential in front of a store raised
// by an earlier installer with the template's. // by an earlier installer with the template's.
// //
// The store's password is set by initdb, once, on an empty volume. A machine that already holds // The store's password is set by initdb, once, on an empty data directory. A machine that already holds
// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new // a cluster in the store's data directory and has no credential file was raised with `bootstrap`, and minting a new
// value here would make a bundle that dials with a password the server does not have — failing // value here would make a bundle that dials with a password the server does not have — failing
// three steps later, in the schemas, with nothing pointing back here. Refused by name instead, // three steps later, in the schemas, with nothing pointing back here. Refused by name instead,
// with the way forward. // with the way forward.
@@ -278,11 +281,14 @@ func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) e
if !c.StoreMade { if !c.StoreMade {
return nil return nil
} }
if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil { // Where the store keeps its data, as the postgres module does (novox/hq ADR 0338 §2): a cluster
return nil // no such volume: a fresh machine, which is the case this installer makes // there is one initdb already set a password for. The docker volume genesis used before is not
// the store's data any more, and a machine holding only that is not refused.
if _, err := run(ctx, "test", "-e", StoreDataDir+"/pgdata/PG_VERSION"); err != nil {
return nil // no cluster there: a fresh machine, which is the case this installer makes
} }
return fmt.Errorf( return fmt.Errorf(
"this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+ "this machine already holds the store's data (in "+StoreDataDir+") and no credential at %s, so it was raised "+
"by an earlier installer with the template's password. A new one made here would not open it. "+ "by an earlier installer with the template's password. A new one made here would not open it. "+
"Put the password the store has into %s (0600, the value alone) and run again; then change it "+ "Put the password the store has into %s (0600, the value alone) and run again; then change it "+
"on the server and accept the new value — this installer does not rotate a running store", "on the server and accept the new value — this installer does not rotate a running store",
@@ -311,7 +317,7 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
return out, err return out, err
} }
if bundle, err = replaceOnce(bundle, templateStoreVolumes, if bundle, err = replaceOnce(bundle, templateStoreVolumes,
`"volumes": ["mesh-store-data:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`, `"volumes": ["`+StoreDataDir+`:/var/lib/postgresql/data", "`+StoreSuperuserFile+":"+storeSuperuserMount+`:ro"]`,
"the store's volumes"); err != nil { "the store's volumes"); err != nil {
return out, err return out, err
} }
+1 -1
View File
@@ -59,7 +59,7 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
switch x := res.(type) { switch x := res.(type) {
case *declaration.Container: case *declaration.Container:
switch x.Name { switch x.Name {
case "mesh-store": case "postgres":
store = true store = true
if _, has := x.Env["POSTGRES_PASSWORD"]; has { if _, has := x.Env["POSTGRES_PASSWORD"]; has {
t.Error("the store still takes its password from its environment") t.Error("the store still takes its password from its environment")
+163
View File
@@ -0,0 +1,163 @@
package bootstrap
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
)
// Defends novox/hq ADR 0338 (issue 609): genesis raises its store as the postgres module's server, so
// the store step adopts it as it runs. Until then the bundle raised `mesh-store`, on another image, in a
// docker volume, and genesis stopped at step 17 of 24.
// theStoreGenesisRaises is the store container of the bundle as the installer applies it: rewritten for
// this machine's credentials, so its password is the file the module reads too.
func theStoreGenesisRaises(t *testing.T) *declaration.Container {
t.Helper()
r := producedBundle(t)
store, err := storeIn(r.Declaration)
if err != nil {
t.Fatal(err)
}
return store
}
// The bundle's store is exactly the catalogue's postgres module's server, read from the module's own
// manifest (ADR 0338 §1): its name, its image (so a drift of either pin fails here), where its data is,
// and the file its superuser password is read from, with the environment that says so.
func TestTheBundleRaisesTheStoreAsTheCataloguesPostgresModuleDeclaresIt(t *testing.T) {
manifest := theCataloguesModule(t, "postgres")
var m struct {
Resources []struct {
ID string `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Image string `json:"image"`
Env map[string]string `json:"env"`
Ports []string `json:"ports"`
Volumes []string `json:"volumes"`
} `json:"resources"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
t.Fatal(err)
}
ours := theStoreGenesisRaises(t)
found := false
for _, r := range m.Resources {
if r.Type != "container" || r.ID != "server" {
continue
}
found = true
if ours.Name != r.Name {
t.Errorf("the bundle raises the store as %q; the module's server is %q", ours.Name, r.Name)
}
if ours.Image != r.Image {
t.Errorf("the bundle raises the store on %s; the module pins %s — the two pins move together", ours.Image, r.Image)
}
for _, key := range []string{"POSTGRES_PASSWORD_FILE", "PGDATA"} {
if ours.Env[key] != r.Env[key] {
t.Errorf("the bundle's store has %s=%q; the module's %q", key, ours.Env[key], r.Env[key])
}
}
if _, has := ours.Env["POSTGRES_PASSWORD"]; has {
t.Error("the bundle's store takes its password from its environment; the module reads a file")
}
// The module's own directory for its state is <data root>/<module>, where genesis keeps the
// superuser's file (StoreSuperuserFile).
var theirs []string
for _, v := range r.Volumes {
theirs = append(theirs, strings.ReplaceAll(v, "${dir:state}", "/var/lib/postgres"))
}
if strings.Join(ours.Volumes, " ") != strings.Join(theirs, " ") {
t.Errorf("the bundle's store mounts %v; the module's %v", ours.Volumes, theirs)
}
if strings.Join(ours.Ports, " ") != strings.Join(r.Ports, " ") {
t.Errorf("the bundle publishes the store on %v; the module on %v", ours.Ports, r.Ports)
}
// And the store step's own check agrees: by name and image, before anything is registered.
if err := serverMatchesFoundation(manifest, ours, "postgres"); err != nil {
t.Errorf("the store step would refuse the store the bundle raises: %v", err)
}
}
if !found {
t.Fatal("the postgres module declares no server container")
}
}
// The store's data directory is made by the bundle before the store starts, owned by the image's
// postgres user, as the module declares it (ADR 0338 §1, and the measurement under its §5).
func TestTheBundleMakesTheStoresDataDirectoryAsTheModuleDoes(t *testing.T) {
r := producedBundle(t)
at, store := -1, -1
for i, res := range r.Declaration.Resources {
if d, ok := res.(*declaration.Directory); ok && d.Path == StoreDataDir {
at = i
if d.Mode != "0700" || d.Owner != "999:70" {
t.Errorf("the store's data directory is made %s %s; the module makes it 0700 999:70", d.Mode, d.Owner)
}
}
if res.Identity() == StoreID {
store = i
}
}
if at < 0 || store < 0 || at > store {
t.Fatalf("the store's data directory %s is not made before the store (directory at %d, store at %d)", StoreDataDir, at, store)
}
}
// No step of the bundle and no step of the installer names `mesh-store`, which is the seat's name only
// (ADR 0338 §2): every step that reaches the store reaches the container the bundle raised.
func TestNoStepReachesTheStoreByTheSeatsName(t *testing.T) {
text := string(theRealBundle(t))
for _, by := range []string{`"in": "mesh-store"`, `container:mesh-store`, `exec mesh-store`, `"name": "mesh-store"`, `mesh-store-data`} {
if strings.Contains(text, by) {
t.Errorf("the bundle still reaches the store as %s", by)
}
}
store := theStoreGenesisRaises(t)
if foundationStore != store.Name {
t.Errorf("the packages step reaches the store as %q; the bundle raises it as %q", foundationStore, store.Name)
}
r := producedBundle(t)
for _, res := range r.Declaration.Resources {
if a, ok := res.(*declaration.Action); ok && a.In != "" && a.In != store.Name && strings.Contains(a.ID, "database") {
t.Errorf("the step %s reaches the store in %q", a.ID, a.In)
}
}
}
// The installer's refusal to raise a store over data it cannot open looks where the store keeps its data
// (ADR 0338 §2): a cluster there and no credential is refused; the old docker volume alone is not.
func TestTheRefusalOverExistingDataLooksInTheStoresDirectory(t *testing.T) {
made := RootCredentials{Store: "s", Bus: "b", StoreMade: true}
machine := func(cluster, volume bool) Runner {
return func(_ context.Context, name string, args ...string) (string, error) {
joined := name + " " + strings.Join(args, " ")
switch {
case joined == "test -e "+StoreDataDir+"/pgdata/PG_VERSION":
if cluster {
return "", nil
}
case strings.HasPrefix(joined, "docker volume inspect"):
if volume {
return "[]", nil
}
}
return "", errors.New("exit status 1")
}
}
err := RefuseExistingServers(context.Background(), machine(true, false), made)
if err == nil || !strings.Contains(err.Error(), StoreDataDir) {
t.Fatalf("a cluster in %s and no credential was not refused naming it: %v", StoreDataDir, err)
}
if err := RefuseExistingServers(context.Background(), machine(false, true), made); err != nil {
t.Fatalf("a machine holding only the old volume was refused: %v", err)
}
if err := RefuseExistingServers(context.Background(), machine(true, false), RootCredentials{Store: "s"}); err != nil {
t.Fatalf("a kept credential was refused: %v", err)
}
}
+3 -3
View File
@@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) {
// run-once and scheduled shapes are excluded on purpose — a step that has exited is not a // run-once and scheduled shapes are excluded on purpose — a step that has exited is not a
// fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the // fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the
// plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all. // plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all.
want := []string{"mesh-store", "mesh-broker", "temp-mesh-controller"} want := []string{"postgres", "mesh-broker-nats", "temp-mesh-controller"}
if len(verified.Running) != len(want) { if len(verified.Running) != len(want) {
t.Fatalf("confirmed %v running, want %v", verified.Running, want) t.Fatalf("confirmed %v running, want %v", verified.Running, want)
} }
@@ -132,7 +132,7 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
// than being told only that something is not what it should be. // than being told only that something is not what it should be.
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) { func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
runtime := &asked{answer: func(_ string, args []string) (string, error) { runtime := &asked{answer: func(_ string, args []string) (string, error) {
if args[0] == "container" && args[len(args)-1] == "mesh-broker" { if args[0] == "container" && args[len(args)-1] == "mesh-broker-nats" {
return "false exited\n", nil return "false exited\n", nil
} }
if args[0] == "container" { if args[0] == "container" {
@@ -146,7 +146,7 @@ func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("a container that had exited was reported as part of a running foundation") t.Fatal("a container that had exited was reported as part of a running foundation")
} }
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") { if !strings.Contains(err.Error(), "mesh-broker-nats") || !strings.Contains(err.Error(), "exited") {
t.Errorf("the failure does not say which container is in what state: %v", err) t.Errorf("the failure does not say which container is in what state: %v", err)
} }
} }
+92
View File
@@ -0,0 +1,92 @@
package outward
import (
"errors"
"fmt"
"net"
"net/netip"
"sort"
)
// Network is one network this machine is directly attached to, and the link it is on (novox/hq ADR
// 0340 §7): the address and prefix of the network, not of the machine's own address on it.
//
// **Every one is said, and the machine judges none of them.** Which count as the machine's local
// network — private ranges only, never a public subnet, never a link-local one — is the controller's
// judgement (ADR 0340 §2), made where the filter is composed and shown with why; a machine that left a
// network out would make that judgement silently.
type Network struct {
Link string `json:"link"`
Network string `json:"network"`
}
// AddressesOf is the addresses on one link, as address and prefix. A parameter so a test can hold a
// machine's links without one.
type AddressesOf func(link string) ([]netip.Prefix, error)
// RealAddresses reads one link's addresses from the kernel.
func RealAddresses(link string) ([]netip.Prefix, error) {
iface, err := net.InterfaceByName(link)
if err != nil {
return nil, err
}
addrs, err := iface.Addrs()
if err != nil {
return nil, err
}
var out []netip.Prefix
for _, a := range addrs {
ipnet, ok := a.(*net.IPNet)
if !ok {
continue
}
addr, ok := netip.AddrFromSlice(ipnet.IP)
if !ok {
continue
}
bits, _ := ipnet.Mask.Size()
out = append(out, netip.PrefixFrom(addr.Unmap(), bits))
}
return out, nil
}
// Networks is the networks on each of these links — the links that face outside (Links) — sorted by link
// and network, without repeats. A link that has gone since Links read it has no networks rather than
// failing the rest; a link whose addresses cannot be read fails, so the machine says the fact is unknown
// rather than reporting a partial account as the whole.
func Networks(links []string, addressesOf AddressesOf) ([]Network, error) {
if addressesOf == nil {
addressesOf = RealAddresses
}
seen := map[Network]bool{}
var out []Network
for _, link := range links {
prefixes, err := addressesOf(link)
if err != nil {
if isNoSuchInterface(err) {
continue
}
return nil, fmt.Errorf("cannot read the addresses on %s: %w", link, err)
}
for _, p := range prefixes {
n := Network{Link: link, Network: p.Masked().String()}
if !seen[n] {
seen[n] = true
out = append(out, n)
}
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Link != out[j].Link {
return out[i].Link < out[j].Link
}
return out[i].Network < out[j].Network
})
return out, nil
}
// isNoSuchInterface is the error net.InterfaceByName gives for a link that is not there.
func isNoSuchInterface(err error) bool {
var op *net.OpError
return errors.As(err, &op) && op.Err != nil && op.Err.Error() == "no such network interface"
}
+59
View File
@@ -0,0 +1,59 @@
package outward
import (
"errors"
"net/netip"
"reflect"
"testing"
)
func prefixes(s ...string) []netip.Prefix {
var out []netip.Prefix
for _, p := range s {
out = append(out, netip.MustParsePrefix(p))
}
return out
}
// **Each link facing outside says the networks it is attached to** (novox/hq ADR 0340 §7), as the
// network's address and prefix, never the machine's own address on it — and every one, public and
// link-local included, because which count is the controller's judgement and said with why.
func TestEachOutwardLinkSaysTheNetworksItIsOn(t *testing.T) {
addrs := map[string][]netip.Prefix{
"eth0": prefixes("192.168.1.10/24", "fe80::1/64", "fd12:3456:789a::5/64"),
"eth1": prefixes("203.0.113.7/24"),
// Not asked: the bridge is not a link facing outside.
"docker0": prefixes("172.17.0.1/16"),
}
got, err := Networks([]string{"eth1", "eth0"}, func(link string) ([]netip.Prefix, error) { return addrs[link], nil })
if err != nil {
t.Fatal(err)
}
want := []Network{
{Link: "eth0", Network: "192.168.1.0/24"},
{Link: "eth0", Network: "fd12:3456:789a::/64"},
{Link: "eth0", Network: "fe80::/64"},
{Link: "eth1", Network: "203.0.113.0/24"},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %+v\nwant %+v", got, want)
}
}
// A link whose addresses cannot be read fails the whole, so the machine says the fact is unknown rather
// than reporting part of its networks as all of them.
func TestAnUnreadableLinkIsNotAPartialAccount(t *testing.T) {
_, err := Networks([]string{"eth0"}, func(string) ([]netip.Prefix, error) { return nil, errors.New("denied") })
if err == nil {
t.Fatal("an unreadable link was reported as having no networks")
}
}
// A link gone between reading the routing table and reading its addresses has no networks, and the
// rest are still said.
func TestALinkGoneSinceItWasListedHasNoNetworks(t *testing.T) {
got, err := Networks([]string{"gone0"}, RealAddresses)
if err != nil || len(got) != 0 {
t.Fatalf("got %v, %v", got, err)
}
}
+35 -2
View File
@@ -10,6 +10,8 @@ import (
"sync" "sync"
"syscall" "syscall"
"time" "time"
"github.com/novox/mesh-host/internal/outward"
) )
// Machine is what this machine is made of, as it says itself (novox/hq issue 160): what somebody // Machine is what this machine is made of, as it says itself (novox/hq issue 160): what somebody
@@ -50,6 +52,11 @@ type Machine struct {
// Filesystems is each local filesystem mounted, by where, with its size and what is free. // Filesystems is each local filesystem mounted, by where, with its size and what is free.
Filesystems []Filesystem `json:"filesystems,omitempty"` Filesystems []Filesystem `json:"filesystems,omitempty"`
// Networks is every network the machine is directly attached to on a link facing outside it, as the
// link and the network's address and prefix (novox/hq ADR 0340 §7): what a port reached from
// `local-network` is resolved from. Every one is said; which count is the controller's judgement.
Networks []outward.Network `json:"networks,omitempty"`
// Unknown names each fact that could not be determined, and why. // Unknown names each fact that could not be determined, and why.
Unknown map[string]string `json:"unknown,omitempty"` Unknown map[string]string `json:"unknown,omitempty"`
} }
@@ -79,11 +86,12 @@ const (
FactTimezone = "timezone" FactTimezone = "timezone"
FactFilesystems = "filesystems" FactFilesystems = "filesystems"
FactPublicAddress = "public address" FactPublicAddress = "public address"
FactNetworks = "networks"
) )
// MachineFacts is every fact Unknown may name, in the order a reader shows them. // MachineFacts is every fact Unknown may name, in the order a reader shows them.
var MachineFacts = []string{FactKernelRelease, FactDistribution, FactVersion, FactProcessor, FactCores, var MachineFacts = []string{FactKernelRelease, FactDistribution, FactVersion, FactProcessor, FactCores,
FactThreads, FactMemory, FactVirtual, FactUptime, FactTimezone, FactFilesystems, FactPublicAddress} FactThreads, FactMemory, FactVirtual, FactUptime, FactTimezone, FactFilesystems, FactNetworks, FactPublicAddress}
// publicAddressUnknown is why the public uplink address is not said. A machine behind a NAT cannot see // publicAddressUnknown is why the public uplink address is not said. A machine behind a NAT cannot see
// its public address from itself, and asking a service outside the mesh is a dependency the mesh does not // its public address from itself, and asking a service outside the mesh is a dependency the mesh does not
@@ -101,11 +109,24 @@ type Sources struct {
// Statfs is (size, free) in bytes of the filesystem mounted at a path. // Statfs is (size, free) in bytes of the filesystem mounted at a path.
Statfs func(path string) (size, free int64, err error) Statfs func(path string) (size, free int64, err error)
Now func() time.Time Now func() time.Time
// Networks is the networks on the links facing outside this machine (novox/hq ADR 0340).
Networks func() ([]outward.Network, error)
} }
// RealSources reads this machine. // RealSources reads this machine.
func RealSources() Sources { func RealSources() Sources {
return Sources{Read: os.ReadFile, Readlink: os.Readlink, Run: ExecRunner, Statfs: statfs, Now: time.Now} return Sources{Read: os.ReadFile, Readlink: os.Readlink, Run: ExecRunner, Statfs: statfs, Now: time.Now,
Networks: attachedNetworks}
}
// attachedNetworks is this machine's networks on the links that face outside it, read the way the report
// reads those links (novox/hq ADR 0140), so the two never disagree about which links those are.
func attachedNetworks() ([]outward.Network, error) {
links, err := outward.Links("", "")
if err != nil {
return nil, err
}
return outward.Networks(links, nil)
} }
func statfs(path string) (int64, int64, error) { func statfs(path string) (int64, int64, error) {
@@ -193,6 +214,18 @@ func DescribeMachine(ctx context.Context, src Sources) Machine {
} }
} }
// The networks it is attached to, for the local network (novox/hq ADR 0340): every one on a link facing
// outside, or why none is said — never an empty list that reads as a machine on no network.
if src.Networks == nil {
unknown(FactNetworks, "this node-engine was given no way to read its links")
} else if networks, err := src.Networks(); err != nil {
unknown(FactNetworks, err.Error())
} else if len(networks) == 0 {
unknown(FactNetworks, "no link facing outside this machine carries an address")
} else {
m.Networks = networks
}
unknown(FactPublicAddress, publicAddressUnknown) unknown(FactPublicAddress, publicAddressUnknown)
return m return m
} }
+29
View File
@@ -10,6 +10,8 @@ import (
"sync/atomic" "sync/atomic"
"testing" "testing"
"time" "time"
"github.com/novox/mesh-host/internal/outward"
) )
// The parsing layer, with the machine's files given (novox/hq ADR 0017 allows a fake here and only here); // The parsing layer, with the machine's files given (novox/hq ADR 0017 allows a fake here and only here);
@@ -40,6 +42,9 @@ func given(read map[string]string) Sources {
Run: func(context.Context, string, ...string) (string, error) { return "none\n", errors.New("exited 1") }, Run: func(context.Context, string, ...string) (string, error) { return "none\n", errors.New("exited 1") },
Statfs: func(p string) (int64, int64, error) { return 1000, 400, nil }, Statfs: func(p string) (int64, int64, error) { return 1000, 400, nil },
Now: func() time.Time { return time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC) }, Now: func() time.Time { return time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC) },
Networks: func() ([]outward.Network, error) {
return []outward.Network{{Link: "eth0", Network: "192.168.1.0/24"}, {Link: "eth1", Network: "203.0.113.0/24"}}, nil
},
} }
} }
@@ -92,6 +97,7 @@ func TestMachine_aFactItCannotReadIsSaidUnknownNeverZero(t *testing.T) {
src := given(map[string]string{}) src := given(map[string]string{})
src.Readlink = func(string) (string, error) { return "", os.ErrNotExist } src.Readlink = func(string) (string, error) { return "", os.ErrNotExist }
src.Run = func(context.Context, string, ...string) (string, error) { return "", errors.New("not found") } src.Run = func(context.Context, string, ...string) (string, error) { return "", errors.New("not found") }
src.Networks = func() ([]outward.Network, error) { return nil, errors.New("cannot read the routing table") }
m := DescribeMachine(context.Background(), src) m := DescribeMachine(context.Background(), src)
raw, _ := json.Marshal(m) raw, _ := json.Marshal(m)
@@ -183,3 +189,26 @@ func TestAgainstThisMachine_machineSaysWhatItIs(t *testing.T) {
raw, _ := json.MarshalIndent(m, "", " ") raw, _ := json.MarshalIndent(m, "", " ")
t.Logf("%s", raw) t.Logf("%s", raw)
} }
// **The networks it is attached to are among its facts** (novox/hq ADR 0340 §7), every one — the public
// one too, which the controller judges and never counts — and a machine with none on a link facing
// outside says why rather than reporting an empty list.
func TestMachine_saysTheNetworksItIsAttachedTo(t *testing.T) {
m := DescribeMachine(context.Background(), given(files))
raw, _ := json.Marshal(m)
var said struct {
Networks []map[string]string `json:"networks"`
}
_ = json.Unmarshal(raw, &said)
if len(said.Networks) != 2 || said.Networks[0]["link"] != "eth0" || said.Networks[0]["network"] != "192.168.1.0/24" ||
said.Networks[1]["network"] != "203.0.113.0/24" {
t.Fatalf("networks said as %s", raw)
}
src := given(files)
src.Networks = func() ([]outward.Network, error) { return nil, nil }
m = DescribeMachine(context.Background(), src)
if len(m.Networks) != 0 || !strings.Contains(m.Unknown[FactNetworks], "no link facing outside") {
t.Fatalf("a machine with no network said %v, unknown %q", m.Networks, m.Unknown[FactNetworks])
}
}