jschoubben
b72b71a989
031: a window of unacknowledged declarations is drained to the newest; the rest are set aside and reported as superseded. 035: a file resource may say create-once — written when absent, kept untouched when present (ADR 0087). 054: the bundle installs nftables and loads a base ruleset before the store and broker, in the table the filter module later replaces (ADR 0088).
185 lines
11 KiB
Plaintext
185 lines
11 KiB
Plaintext
// foundation-first-node.lock — what a machine must be before a mesh exists.
|
|
//
|
|
// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
|
|
// store, a database per context, those contexts' schemas, the broker, and the control plane
|
|
// running on top of them.
|
|
//
|
|
// It stopped before the control plane once, and this comment said so for longer than it was true.
|
|
//
|
|
// The broker generates its OWN certificate, in its own image, into a volume it then mounts read
|
|
// only. Self-signed, because at this moment there is no mesh to issue one and no public name to
|
|
// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint
|
|
// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration.
|
|
//
|
|
// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
|
|
// image, and this file is applied on a machine with no mesh to ask about anything. These digests
|
|
// belong to the registry the lab raises, which is what a real node pulls from anyway — what is
|
|
// required is a reference that is exact and cannot move (novox/hq ADR 0006).
|
|
//
|
|
// The store waits up to three minutes rather than one. A machine that has just pulled the
|
|
// image and is running initdb for the first time can take longer than sixty seconds, and it
|
|
// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed,
|
|
// which is the worst kind because it teaches people to run things twice.
|
|
//
|
|
// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited
|
|
// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes
|
|
// a retry the only available response is a timeout that reports nothing. So the wait now says
|
|
// what it saw before giving up.
|
|
//
|
|
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
|
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
|
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
|
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
|
// state.
|
|
{
|
|
"declaration": 1,
|
|
"resources": [
|
|
{
|
|
"id": "container-runtime",
|
|
"type": "package",
|
|
"package": "docker"
|
|
},
|
|
{
|
|
"id": "container-runtime-running",
|
|
"type": "service",
|
|
"unit": "docker.service",
|
|
"state": "running",
|
|
"boot": "enabled"
|
|
},
|
|
// **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
|
|
// broker are adopted as modules later and so bind to every interface from the moment they
|
|
// start; the packet filter that governs who may reach them is a module too, installed a
|
|
// dozen steps later. Between the two, a control-node facing the network had its store and
|
|
// its bus open to anyone who could reach the machine. So the foundation carries a filter of
|
|
// its own — the same table the filter module will replace wholesale once it can derive one:
|
|
// drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
|
|
// enrols over, the registry a node pulls from), and let the container runtime's own
|
|
// networks through the forward chain so containers keep working. A published container port
|
|
// is forwarded, never input (issue 047), which is why the forward chain is where the store's
|
|
// and broker's ports are refused from outside.
|
|
{
|
|
"id": "base-filter-package",
|
|
"type": "package",
|
|
"package": "nftables"
|
|
},
|
|
{
|
|
"id": "base-filter",
|
|
"type": "file",
|
|
"path": "/etc/nftables.conf",
|
|
"mode": "0644",
|
|
"content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
|
|
},
|
|
{
|
|
"id": "base-filter-loaded",
|
|
"type": "service",
|
|
"unit": "nftables.service",
|
|
"state": "running",
|
|
"boot": "enabled",
|
|
"restart-on": ["base-filter"]
|
|
},
|
|
{
|
|
"id": "store",
|
|
"type": "container",
|
|
"name": "mesh-store",
|
|
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
|
"env": {
|
|
"POSTGRES_PASSWORD": "bootstrap",
|
|
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
|
},
|
|
"ports": ["5432:5432"],
|
|
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
|
},
|
|
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
|
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
|
// exits happy, and the verify a moment later lands in the gap and fails. The action and its
|
|
// verify must ask the same question, or the action can succeed into a state verify rejects.
|
|
{
|
|
"id": "store-ready",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
|
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
|
},
|
|
{
|
|
"id": "inventory-database",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
|
},
|
|
{
|
|
"id": "identity-database",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
|
},
|
|
// Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
|
|
// created the same way and named the same way — which is the whole of adding a context to the
|
|
// bootstrap, and is why the count is not something the foundation has an opinion about.
|
|
{
|
|
"id": "licences-database",
|
|
"type": "action",
|
|
"in": "mesh-store",
|
|
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
|
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
|
},
|
|
{
|
|
"id": "context-schemas",
|
|
"type": "action",
|
|
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
|
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
|
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
|
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
|
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
"migrate"],
|
|
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
|
},
|
|
{
|
|
"id": "broker-certificate",
|
|
"type": "action",
|
|
"command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
|
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
|
"-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
|
|
"verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
|
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
|
"-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
|
|
},
|
|
{
|
|
"id": "broker",
|
|
"type": "container",
|
|
"name": "mesh-broker",
|
|
"image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
|
"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"],
|
|
"volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"],
|
|
"args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"]
|
|
},
|
|
{
|
|
"id": "broker-ready",
|
|
"type": "action",
|
|
"in": "mesh-broker",
|
|
"command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
|
"verify": ["lavinmqctl", "status"]
|
|
},
|
|
{
|
|
"id": "control-plane",
|
|
"type": "container",
|
|
"name": "mesh-controller",
|
|
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
|
"network": "host",
|
|
"args": ["serve"],
|
|
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
|
"env": {
|
|
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
|
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
|
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
|
"MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/",
|
|
"MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672",
|
|
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
|
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
|
}
|
|
}
|
|
|
|
]
|
|
}
|