jschoubben
971a6d6d03
novox/hq 04-ISSUES/146. Each was right while the mesh ran on the previous broker, and nothing has raised a foundation since it changed. The bus's certificate is made by the program that needs it rather than by openssl inside the broker's image — the bus's image is Alpine with a shell and no openssl, so the step exited 127 and no mesh could be raised. Self-signed as before and on purpose; --user 0:0 because the volume is root's and the control plane's image runs as nobody. Enrolment no longer opens a raw TLS connection to check the pin: NATS speaks its own protocol and upgrades afterwards, so the handshake met a plaintext greeting. The client that presents the token carries the same pinned config and verifies inside its own handshake, so the secret still leaves only after the certificate is checked. The raw dial stays as what its tests prove, and is no longer a path anything takes. And the token says which bus it is for. Empty meant 'whatever the mesh runs today' while two buses existed and became a refusal the moment one did. It now stops at the bus's user list, which is the genesis half of 146.
201 lines
13 KiB
Plaintext
201 lines
13 KiB
Plaintext
// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
|
|
// built (novox/hq ADR 0106, design 25).
|
|
//
|
|
// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
|
|
// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
|
|
// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
|
|
// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
|
|
// from the controller's first composition onward the file is the controller's to write.
|
|
//
|
|
// The accounts file is its own file beside the server's configuration, because the server's own
|
|
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
|
// one directory, of necessity: an include path is resolved relative to the including file's own
|
|
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
|
// start.
|
|
//
|
|
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
|
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
|
// authenticates with a password (ADR 0004, design 25 §4).
|
|
|
|
{
|
|
"declaration": 1,
|
|
"resources": [
|
|
{
|
|
"id": "container-runtime",
|
|
"type": "package",
|
|
"package": "docker"
|
|
},
|
|
{
|
|
"id": "container-runtime-running",
|
|
"type": "service",
|
|
"unit": "docker.service",
|
|
"state": "running",
|
|
"boot": "enabled"
|
|
},
|
|
// **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
|
|
// broker are adopted as modules later and so bind to every interface from the moment they
|
|
// start; the packet filter that governs who may reach them is a module too, installed a
|
|
// dozen steps later. Between the two, a control-node facing the network had its store and
|
|
// its bus open to anyone who could reach the machine. So the foundation carries a filter of
|
|
// its own — the same table the filter module will replace wholesale once it can derive one:
|
|
// drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
|
|
// enrols over, the registry a node pulls from), and let the container runtime's own
|
|
// networks through the forward chain so containers keep working. A published container port
|
|
// is forwarded, never input (issue 047), which is why the forward chain is where the store's
|
|
// and broker's ports are refused from outside — and a container on this machine dialling a
|
|
// port this machine publishes reaches it through the runtime's proxy, which IS input, which
|
|
// is why the bus and the registry are opened in both chains, exactly as the derived ruleset
|
|
// does.
|
|
{
|
|
"id": "base-filter-package",
|
|
"type": "package",
|
|
"package": "nftables"
|
|
},
|
|
{
|
|
"id": "base-filter",
|
|
"type": "file",
|
|
"path": "/etc/nftables.conf",
|
|
"mode": "0644",
|
|
"content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
|
|
},
|
|
{
|
|
"id": "base-filter-loaded",
|
|
"type": "service",
|
|
"unit": "nftables.service",
|
|
"state": "running",
|
|
"boot": "enabled",
|
|
"restart-on": ["base-filter"]
|
|
},
|
|
{
|
|
"id": "store",
|
|
"type": "container",
|
|
"name": "mesh-store",
|
|
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
|
"env": {
|
|
"POSTGRES_PASSWORD": "bootstrap",
|
|
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
|
},
|
|
"ports": ["5432:5432"],
|
|
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
|
},
|
|
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
|
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
|
// exits happy, and the verify a moment later lands in the gap and fails. The action and its
|
|
// verify must ask the same question, or the action can succeed into a state verify rejects.
|
|
{
|
|
"id": "store-ready",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
|
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
|
},
|
|
{
|
|
"id": "inventory-database",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
|
},
|
|
{
|
|
"id": "identity-database",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
|
},
|
|
// Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
|
|
// created the same way and named the same way — which is the whole of adding a context to the
|
|
// bootstrap, and is why the count is not something the foundation has an opinion about.
|
|
{
|
|
"id": "licences-database",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
|
},
|
|
{
|
|
"id": "context-schemas",
|
|
"type": "action",
|
|
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
|
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
|
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
|
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
"migrate"],
|
|
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
|
},
|
|
{
|
|
"id": "bus-certificate",
|
|
"type": "action",
|
|
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
|
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
|
// shell and no openssl, and no other image the bundle names has one either. So the program
|
|
// that needs the certificate writes it — already on this machine, since the schema step ran
|
|
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
|
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
|
// to ask an authority of.
|
|
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right
|
|
// for the long-running control plane and wrong for a one-shot writing into a fresh volume.
|
|
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
"broker", "certificate", "--into", "/tls"],
|
|
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
"broker", "certificate", "--check", "--into", "/tls"]
|
|
},
|
|
{
|
|
"id": "bus-conf-dir",
|
|
"type": "directory",
|
|
"path": "/var/lib/mesh-bus-conf",
|
|
"mode": "0700"
|
|
},
|
|
{
|
|
"id": "bus-conf",
|
|
"type": "file",
|
|
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
|
"mode": "0644",
|
|
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
|
},
|
|
{
|
|
"id": "bus-accounts",
|
|
"type": "file",
|
|
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
|
"mode": "0600",
|
|
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.API.>\", \"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"_INBOX.enrol.>\", \"mesh.control.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\"] }\n subscribe: { allow: [\"$JS.API.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.seat.mesh-build-machine.event.built\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
|
},
|
|
{
|
|
"id": "broker",
|
|
"type": "container",
|
|
"name": "mesh-broker",
|
|
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
|
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
|
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
|
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
|
},
|
|
{
|
|
"id": "broker-ready",
|
|
"type": "action",
|
|
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
|
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
|
},
|
|
{
|
|
"id": "control-plane",
|
|
"type": "container",
|
|
"name": "mesh-controller",
|
|
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
"network": "host",
|
|
"args": ["serve"],
|
|
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
|
"env": {
|
|
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
|
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
|
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
|
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
|
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
|
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
|
}
|
|
}
|
|
|
|
]
|
|
}
|