diff --git a/README.md b/README.md index 39a6014..d0b878f 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,18 @@ export MESH_LAB_BUNDLE=/examples/substrate-first-node.lock export MESH_LAB_MODULES=/examples/modules export MESH_LAB_BUILDER=/build/mesh-builder # build/, which is git-ignored +# The installer. `suite` builds it with mesh-host's `make bootstrap`, which embeds a `docker save` +# of the control-plane image — so it is built AFTER that image, in the same run, or it carries a +# stale one sealed inside a binary where nothing would ever notice. The whole-mesh bed raises its +# anchor by RUNNING this, rather than by applying a substrate bundle itself (novox/hq ADR 0067). +export MESH_LAB_BOOTSTRAP_BINARY=/mesh-bootstrap +export MESH_LAB_CONTROL_IMAGE=mesh-control:development # optional; what it carries + +# A checkout of the mesh's catalogue. The installer reads the registry's and the control plane's +# manifests from a copy of it ON THE MACHINE, because at genesis there is no forge, no build +# machine and — until the registry is up — nothing serving anything. +export MESH_LAB_CATALOG=/modules + # Built with `go build -o ./examples/` in mesh-control. export MESH_LAB_PROVISIONER=/postgres-provisioner export MESH_LAB_OBJECTSTORE_PROVISIONER=/objectstore-provisioner @@ -194,7 +206,7 @@ the machines instead: ```sh incus list -c ns -incus exec -registry -- systemctl is-active docker +incus exec -anchor -- systemctl is-active docker ``` *"I cannot see progress" is not evidence of no progress.* diff --git a/scenarios/a-provider.yml b/scenarios/a-provider.yml index f221c17..637dfe8 100644 --- a/scenarios/a-provider.yml +++ b/scenarios/a-provider.yml @@ -13,10 +13,8 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow -images: - - postgres:17-alpine - place: all: [runtime] diff --git a/scenarios/a-public-name.yml b/scenarios/a-public-name.yml index ecb24e8..b039022 100644 --- a/scenarios/a-public-name.yml +++ b/scenarios/a-public-name.yml @@ -17,10 +17,8 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow -images: - - ghcr.io/letsencrypt/pebble:2.5.0 - place: all: [runtime] diff --git a/scenarios/an-object-store.yml b/scenarios/an-object-store.yml index 7145f21..dc23b1b 100644 --- a/scenarios/an-object-store.yml +++ b/scenarios/an-object-store.yml @@ -19,13 +19,8 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow -images: - - minio/minio:RELEASE.2025-09-07T16-13-09Z - # The vendor's client, stocked so the provisioner has the thing it drives without reaching a - # public registry from a documentation range. - - minio/mc:RELEASE.2025-08-13T08-35-41Z - place: all: [runtime] diff --git a/scenarios/anthropic-bed.yml b/scenarios/anthropic-bed.yml index 2ba5c94..02199c6 100644 --- a/scenarios/anthropic-bed.yml +++ b/scenarios/anthropic-bed.yml @@ -29,17 +29,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # The two model-access runtimes, built by scripts/build-module-runtime.sh into the local daemon and - # stocked into the scenario's own registry, which is where the host pulls them from. + # loaded onto the machine, which holds them by their own image IDs. - mesh-runtime-anthropic-manager:development - mesh-runtime-anthropic-consumer:development diff --git a/scenarios/audit-node.yml b/scenarios/audit-node.yml index 958c505..8107155 100644 --- a/scenarios/audit-node.yml +++ b/scenarios/audit-node.yml @@ -14,16 +14,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # The tool runtime with the audit-logger, built by scripts/build-runtime-image.sh into the local - # daemon and stocked into the scenario's own registry, which is where the host pulls it from. + # daemon and loaded onto the machine, which holds it by its own image ID. - mesh-runtime-audit:development place: diff --git a/scenarios/behind-nat.yml b/scenarios/behind-nat.yml index e1b88df..4541a5b 100644 --- a/scenarios/behind-nat.yml +++ b/scenarios/behind-nat.yml @@ -11,7 +11,7 @@ segments: home: kind: private - cidr: [192.168.1.0/24] + cidr: [10.99.1.0/24] gateway: to: hosting address: [192.0.2.50] # what the world sees the household as @@ -25,7 +25,7 @@ machines: inbound: allow home-server: # a dash in the name, on purpose - at: { segment: home, address: [192.168.1.135] } + at: { segment: home, address: [10.99.1.135] } published: - { port: 8080, on: home } inbound: allow diff --git a/scenarios/bootstrap-with-registry.yml b/scenarios/bootstrap-with-registry.yml deleted file mode 100644 index 83a92f0..0000000 --- a/scenarios/bootstrap-with-registry.yml +++ /dev/null @@ -1,23 +0,0 @@ -# One machine and a registry, which is the smallest scenario that can exercise a container. -# -# A sealed machine cannot reach a registry and an image placed from an archive cannot keep its -# digest (novox/hq 04-ISSUES/009), so the lab raises one inside the scenario and serves the -# images below from it. What a declaration pins is reported when this is raised — the digest -# belongs to this registry, not to the one the image came from. -scenario: bootstrap-with-registry - -segments: - hosting: - kind: public - cidr: [192.0.2.0/24] - -machines: - anchor: - at: { segment: hosting, address: [192.0.2.10] } - inbound: allow - -images: - - alpine:3.20 - -place: - all: [host, runtime] diff --git a/scenarios/catalogue-apps.yml b/scenarios/catalogue-apps.yml index 513c87e..9a46c1d 100644 --- a/scenarios/catalogue-apps.yml +++ b/scenarios/catalogue-apps.yml @@ -28,6 +28,7 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow # Sized up past catalogue-small's 6GiB: this wave carries two heavy JVM/embedded-DB service # containers (the UniFi controller and MaryTTS) on top of the substrate, mongodb, postgres and @@ -36,14 +37,7 @@ machines: cpus: 4 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The module server images. - - mongo:7 - - lscr.io/linuxserver/unifi-controller:latest - - synesthesiam/marytts:lab # The runtimes built by scripts/build-module-runtime.sh and stocked here. marrytts needs none. - mesh-runtime-mongodb:development - mesh-runtime-unifi:development diff --git a/scenarios/catalogue-media.yml b/scenarios/catalogue-media.yml index dc70674..9dc1534 100644 --- a/scenarios/catalogue-media.yml +++ b/scenarios/catalogue-media.yml @@ -30,6 +30,7 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow # Two *arr apps (server + runtime each) on top of the first-node substrate — seven containers. # The Servarr images are lighter than catalogue-apps' JVM pair, so catalogue-small's 6GiB is @@ -38,13 +39,7 @@ machines: cpus: 4 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The module server images. - - lscr.io/linuxserver/sonarr:latest - - lscr.io/linuxserver/radarr:latest # The two runtimes built by scripts/build-module-runtime.sh and stocked here. - mesh-runtime-sonarr:development - mesh-runtime-radarr:development diff --git a/scenarios/catalogue-mqtt.yml b/scenarios/catalogue-mqtt.yml index ef47164..866b7c3 100644 --- a/scenarios/catalogue-mqtt.yml +++ b/scenarios/catalogue-mqtt.yml @@ -14,7 +14,7 @@ # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock # scripts/build-module-runtime.sh mosquitto builds mesh-runtime-mosquitto:development (carrying # mosquitto_ctrl and the compiled bootstrap entrypoint) into the local daemon, which this scenario -# stocks and serves by digest from its own registry. eclipse-mosquitto:2 must be in the local +# pulls from the internet over its uplink. eclipse-mosquitto:2 must be in the local # daemon to be stocked. scenario: catalogue-mqtt @@ -26,20 +26,13 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # mosquitto's broker (the service image) and its runtime, the latter built by - # scripts/build-module-runtime.sh mosquitto into the local daemon and stocked into the scenario's - # own registry, which is where the host pulls it from. The runtime image is reused for the - # run-once bootstrap step and for the serve container. - - eclipse-mosquitto:2 - mesh-runtime-mosquitto:development place: diff --git a/scenarios/catalogue-small.yml b/scenarios/catalogue-small.yml index 8c77754..f45c5f1 100644 --- a/scenarios/catalogue-small.yml +++ b/scenarios/catalogue-small.yml @@ -13,7 +13,7 @@ # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock # scripts/build-module-runtime.sh {postgres,redis,minio,plex} build the four runtime images into the # local daemon (postgres carries psql, minio carries mc), which this scenario stocks and serves by -# digest from its own registry. The service images must be in the local daemon to be stocked. +# the internet over its uplink. Only the mesh's own images come from the local daemon. scenario: catalogue-small segments: @@ -24,6 +24,7 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow # Sized up: this anchor runs the substrate (store, broker, control) plus four modules — three of # which are a server container and a runtime container each — so a dozen containers at once. The @@ -32,13 +33,7 @@ machines: cpus: 4 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The module server images. - - redis:7-alpine - - minio/minio:latest # The four per-module runtimes, built by scripts/build-module-runtime.sh and stocked here. plex # needs no server image in the lab — its runtime serves tools with no Plex to reach. - mesh-runtime-postgres:development diff --git a/scenarios/first-node.yml b/scenarios/first-node.yml index 0566555..4065e23 100644 --- a/scenarios/first-node.yml +++ b/scenarios/first-node.yml @@ -1,8 +1,8 @@ # One machine, raising a substrate from the bundle its host carries. # # This is the bootstrap class (novox/hq ADR 0009): no forge, no control plane to talk to, no -# delivery. It exists to develop the steps of raising a mesh on a machine with no route out — -# a container runtime, a store, the control plane's schema in it, and the broker. +# delivery. It exists to develop the steps of raising a mesh on a machine that has nothing but a +# connection — a container runtime, a store, the control plane's schema in it, and the broker. # # It stops before the control plane *runs*, because there is nothing for it to serve yet. scenario: first-node @@ -15,14 +15,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow -# Placed into a registry the scenario raises, which is what a real node pulls from anyway. The -# digests below are the ones that registry assigns, and that satisfies pinning: what is required -# is a reference that is exact and cannot move (novox/hq ADR 0006). +# The control plane's own image exists in no registry — it is built from source, and until this +# mesh has a registry of its own there is nowhere to have pushed it. So it is loaded onto the +# machine and named by the digest of its own configuration, which is exact and cannot move, which +# is what pinning asks for (novox/hq ADR 0006). The store and the broker are ordinary third-party +# images, and the machine pulls them from the internet like anything else. images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development place: diff --git a/scenarios/grafana-node.yml b/scenarios/grafana-node.yml index dac3864..e58814c 100644 --- a/scenarios/grafana-node.yml +++ b/scenarios/grafana-node.yml @@ -14,13 +14,12 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - mesh-runtime-grafana:development diff --git a/scenarios/growing-mesh.yml b/scenarios/growing-mesh.yml index 41fcd55..0ab0f04 100644 --- a/scenarios/growing-mesh.yml +++ b/scenarios/growing-mesh.yml @@ -16,17 +16,18 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow laptop: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow workstation: at: { segment: hosting, address: [192.0.2.30] } + egress: true inbound: allow images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development place: diff --git a/scenarios/lavinmq-bed.yml b/scenarios/lavinmq-bed.yml index ed9d8fd..f20fcdc 100644 --- a/scenarios/lavinmq-bed.yml +++ b/scenarios/lavinmq-bed.yml @@ -22,8 +22,8 @@ # scripts/build-module-runtime.sh lavinmq /tmp/lavinmq.tar # scripts/build-module-runtime.sh amqp-ping /tmp/amqp-ping.tar # The service image cloudamqp/lavinmq:latest must be in the local daemon too — it is already stocked as -# the substrate's own broker image; each node pulls what it runs from the scenario's own registry by -# digest. +# the substrate's own broker image; each node pulls what it runs from the internet, over its own +# uplink. scenario: lavinmq-bed segments: @@ -34,23 +34,22 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 laptop: at: { segment: hosting, address: [192.0.2.11] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker (itself lavinmq), control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # The lavinmq provider's runtime (reused for its run-once bootstrap and its provisioner) and the # amqp-ping consumer's runtime, both built by scripts/build-module-runtime.sh into the local daemon - # and stocked into the scenario's own registry, which is where the hosts pull them from by digest. + # and loaded onto the machines, which hold them by their own image IDs. # The lavinmq SERVICE image is cloudamqp/lavinmq:latest, already stocked above. - mesh-runtime-lavinmq:development - mesh-runtime-amqp-ping:development diff --git a/scenarios/local-model-bed.yml b/scenarios/local-model-bed.yml index f13a9c3..b76b292 100644 --- a/scenarios/local-model-bed.yml +++ b/scenarios/local-model-bed.yml @@ -24,20 +24,14 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 4GiB cpus: 4 disk: 40GiB images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The local model server the ollama provider runs. Pulled by raise() and stocked into the scenario's - # own registry, which is where the host pulls it from. No model is pulled into it — the bed proves the - # mesh routes a consumer to the server's endpoint, not that the server generates tokens. - - ollama/ollama:latest place: all: [host, runtime] diff --git a/scenarios/minio-node.yml b/scenarios/minio-node.yml index 764ad88..6347a7c 100644 --- a/scenarios/minio-node.yml +++ b/scenarios/minio-node.yml @@ -15,17 +15,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - - minio/minio:latest - # minio's runtime, built by scripts/build-module-runtime.sh minio (it carries mc), stocked into the - # scenario's own registry. + # minio's runtime, built by scripts/build-module-runtime.sh minio (it carries mc), loaded onto + # the machine. - mesh-runtime-minio:development place: diff --git a/scenarios/model-usage-bed.yml b/scenarios/model-usage-bed.yml index c428b23..bd067b0 100644 --- a/scenarios/model-usage-bed.yml +++ b/scenarios/model-usage-bed.yml @@ -25,25 +25,23 @@ machines: # The substrate ONLY: store, broker, control — three containers. anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 4GiB cpus: 4 # The postgres PROVIDER (server + broker-bound runtime) and the model-usage CONSUMER (its run-once # migrate and its long-lived event runtime). The 5432-vs-substrate conflict is gone because the # substrate store is on the OTHER node. The runtime images plus postgres:17-alpine are pulled from - # the scenario's own registry by digest; forty gigabytes holds them with room to spare. + # the internet over the uplink; forty gigabytes holds them with room to spare. laptop: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow memory: 4GiB cpus: 4 disk: 40GiB images: - # The first-node substrate: store, broker, control. postgres:17-alpine doubles as postgres's own - # service image. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # The per-module runtimes, built by scripts/build-module-runtime.sh and stocked here. Each carries # its module's code — postgres its provisioner, model-usage its consumer, tools and run-once migrate. diff --git a/scenarios/openai-bed.yml b/scenarios/openai-bed.yml index af5db10..ffac75a 100644 --- a/scenarios/openai-bed.yml +++ b/scenarios/openai-bed.yml @@ -24,17 +24,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # The static-key consumer runtime, built by scripts/build-module-runtime.sh into the local daemon and - # stocked into the scenario's own registry, which is where the host pulls it from. + # loaded onto the machine, which holds it by its own image ID. - mesh-runtime-openai-consumer:development place: diff --git a/scenarios/plex-node.yml b/scenarios/plex-node.yml index 67724b9..dc72b8d 100644 --- a/scenarios/plex-node.yml +++ b/scenarios/plex-node.yml @@ -15,16 +15,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # Plex's tool runtime, built by scripts/build-module-runtime.sh plex into the local daemon and - # stocked into the scenario's own registry, which is where the host pulls it from. + # loaded onto the machine, which holds it by its own image ID. - mesh-runtime-plex:development place: diff --git a/scenarios/postgres-node.yml b/scenarios/postgres-node.yml index 36d7ad0..913fa3f 100644 --- a/scenarios/postgres-node.yml +++ b/scenarios/postgres-node.yml @@ -14,16 +14,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # postgres's runtime, built by scripts/build-module-runtime.sh postgres (it carries psql), stocked - # into the scenario's own registry. + # postgres's runtime, built by scripts/build-module-runtime.sh postgres (it carries psql), loaded + # onto the machine. - mesh-runtime-postgres:development place: diff --git a/scenarios/redis-node.yml b/scenarios/redis-node.yml index 9c44d97..d075371 100644 --- a/scenarios/redis-node.yml +++ b/scenarios/redis-node.yml @@ -14,17 +14,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - - redis:7-alpine # Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local - # daemon and stocked into the scenario's own registry, which is where the host pulls it from. + # daemon and loaded onto the machine, which holds it by its own image ID. - mesh-runtime-redis:development place: diff --git a/scenarios/route-forwarding.yml b/scenarios/route-forwarding.yml index ae25cb5..75da3b8 100644 --- a/scenarios/route-forwarding.yml +++ b/scenarios/route-forwarding.yml @@ -29,21 +29,18 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # The route-proxy's image, built from the canonical Go proxy in mesh-control by # scripts/build-route-proxy-image.sh, and hello-web's backend, a bare alpine nc loop. - mesh-route-proxy:development - - alpine:latest place: # Only the host — neither module carries a mesh-runtime. Both service images are served by the - # scenario's registry and pulled by the host, not placed inside the machine. + # internet and pulled by the host over its uplink, not placed inside the machine. all: [host] diff --git a/scenarios/schedule-tick.yml b/scenarios/schedule-tick.yml index e8edfeb..e345da4 100644 --- a/scenarios/schedule-tick.yml +++ b/scenarios/schedule-tick.yml @@ -15,8 +15,8 @@ # - it fires AGAIN on the following minute — recurrence, not a one-shot. # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_MODULES=.../mesh-control/examples/modules -# alpine:latest must be in the local daemon; the scenario stocks it into its own registry and -# serves it by digest, which is what the scheduled container declares (via pinned("alpine")). +# alpine:latest must be in the local daemon; the machine pulls it from the internet over its +# uplink, and the scheduled container declares it exactly as the catalogue writes it. # There is no runtime image: schedtest carries no code of its own — the scheduled container is a # bare alpine that runs `date >> /data/runs.log` and exits. scenario: schedule-tick @@ -29,21 +29,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The tick container's image. schedtest has no runtime of its own — its scheduled container is a - # bare alpine that appends a timestamp and exits. alpine:latest must be in the local daemon; the - # scenario stocks it and serves it by digest, which is what the manifest pins via pinned("alpine"). - - alpine:latest place: - # Only the host — schedtest has no mesh-runtime to place. The tick image is served by the - # scenario's registry and pulled by the host, not placed inside the machine. + # Only the host — schedtest has no mesh-runtime to place. The tick image is pulled from the + # internet by the host over its uplink, not placed inside the machine. all: [host] diff --git a/scenarios/segmented-and-unforwardable.yml b/scenarios/segmented-and-unforwardable.yml index b7cf4b7..f8dd616 100644 --- a/scenarios/segmented-and-unforwardable.yml +++ b/scenarios/segmented-and-unforwardable.yml @@ -15,7 +15,7 @@ segments: home: kind: private - cidr: [192.168.1.0/24] + cidr: [10.99.1.0/24] gateway: to: hosting address: [192.0.2.50] @@ -53,7 +53,7 @@ machines: inbound: allow home-server: - at: { segment: home, address: [192.168.1.135] } + at: { segment: home, address: [10.99.1.135] } inbound: allow thermostat: diff --git a/scenarios/sonarr-node.yml b/scenarios/sonarr-node.yml index 85dcaf9..8313f6e 100644 --- a/scenarios/sonarr-node.yml +++ b/scenarios/sonarr-node.yml @@ -14,13 +14,12 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - mesh-runtime-sonarr:development diff --git a/scenarios/the-ordinary-shape.yml b/scenarios/the-ordinary-shape.yml index 3277605..2e9016f 100644 --- a/scenarios/the-ordinary-shape.yml +++ b/scenarios/the-ordinary-shape.yml @@ -21,7 +21,7 @@ segments: home: kind: private - cidr: [192.168.1.0/24, "2001:db8:b:1::/64"] + cidr: [10.99.1.0/24, "2001:db8:b:1::/64"] mtu: 1492 gateway: to: isp-home @@ -61,17 +61,17 @@ machines: inbound: allow home-server: - at: { segment: home, address: [192.168.1.135, "2001:db8:b:1::135"] } + at: { segment: home, address: [10.99.1.135, "2001:db8:b:1::135"] } published: - { port: 443, on: home } inbound: allow workstation: - at: { segment: home, address: [192.168.1.250, "2001:db8:b:1::250"] } + at: { segment: home, address: [10.99.1.250, "2001:db8:b:1::250"] } inbound: deny laptop: - at: { segment: home, address: [192.168.1.98, "2001:db8:b:1::98"] } + at: { segment: home, address: [10.99.1.98, "2001:db8:b:1::98"] } inbound: deny # No `place:` yet. The node host it would place does not exist — this lab is being built to diff --git a/scenarios/tools-confluence.yml b/scenarios/tools-confluence.yml index 95cb493..7fcc150 100644 --- a/scenarios/tools-confluence.yml +++ b/scenarios/tools-confluence.yml @@ -11,7 +11,7 @@ # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock # scripts/build-module-runtime.sh confluence builds mesh-runtime-confluence:development into the -# local daemon, which this scenario stocks and serves by digest from its own registry. confluence +# local daemon, which the machine pulls from the internet over its uplink. confluence # needs no service image — it is tools-only. scenario: tools-confluence @@ -23,17 +23,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # confluence's runtime, built by scripts/build-module-runtime.sh confluence into the local daemon - # and stocked into the scenario's own registry, which is where the host pulls it from. There is no + # and loaded onto the machine, which holds it by its own image ID. There is no # service image: confluence is tools-only and outbound-only. - mesh-runtime-confluence:development diff --git a/scenarios/tools-gitlab.yml b/scenarios/tools-gitlab.yml index 2473590..e797999 100644 --- a/scenarios/tools-gitlab.yml +++ b/scenarios/tools-gitlab.yml @@ -12,7 +12,7 @@ # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock # scripts/build-module-runtime.sh gitlab builds mesh-runtime-gitlab:development into the local -# daemon, which this scenario stocks and serves by digest from its own registry. gitlab needs no +# daemon, which the machine pulls from the internet over its uplink. gitlab needs no # service image — it is tools-only. scenario: tools-gitlab @@ -24,17 +24,15 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 3GiB cpus: 2 images: - # The first-node substrate: store, broker, control. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development # gitlab's runtime, built by scripts/build-module-runtime.sh gitlab into the local daemon and - # stocked into the scenario's own registry, which is where the host pulls it from. There is no + # loaded onto the machine, which holds it by its own image ID. There is no # service image: gitlab is tools-only and outbound-only. - mesh-runtime-gitlab:development diff --git a/scenarios/two-node-db.yml b/scenarios/two-node-db.yml index 8218926..84b1313 100644 --- a/scenarios/two-node-db.yml +++ b/scenarios/two-node-db.yml @@ -21,6 +21,7 @@ machines: # containers on a node, which this one never does. anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 4GiB cpus: 4 @@ -31,25 +32,18 @@ machines: # so — and convergence would present as "the mesh hangs". Six gigabytes gives it room. laptop: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow memory: 6GiB cpus: 4 # The runtime images (280MB–600MB each) plus the service images — two of them heavy app images - # (Baserow ~1.5GB, Letta ~1.8GB) — are pulled from the scenario's own registry by digest, so the + # (Baserow ~1.5GB, Letta ~1.8GB) — are pulled from the internet over the uplink, so the # same bytes land on this node twice. The pool default root disk exhausts mid-apply ("no space # left on device"); sixty gigabytes holds the whole chain. disk: 60GiB images: - # The first-node substrate: store, broker, control. postgres:17-alpine doubles as postgres's own - # service image. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The module service images. - - redis:7-alpine - - baserow/baserow:latest - - letta/letta:latest # The per-module runtimes, built by scripts/build-module-runtime.sh and stocked here. Each carries # its module's provisioner, so no separate mesh-provision-* image is listed — the runtime is the # provisioner (ADR 0048). diff --git a/scenarios/two-nodes.yml b/scenarios/two-nodes.yml index bbfeb6f..563ea5f 100644 --- a/scenarios/two-nodes.yml +++ b/scenarios/two-nodes.yml @@ -15,6 +15,7 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow # The whole substrate, the registry, the builder, an adopted workload and the modules under # test all land here — eleven containers before the forge arrives. At the 1GiB default this @@ -24,21 +25,12 @@ machines: cpus: 4 laptop: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow memory: 2GiB images: - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # So a module can mirror one into a registry of the mesh's own. The scenario's registry serves - # what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved - # the same way. - - registry:2 - # A real third-party workload, for adopting one the way the conversion will. Its database is - # the substrate's postgres image rather than its own: what is under test is the mesh delivering - # a module, not which postgres it delivers. - - ghcr.io/umami-software/umami:postgresql-latest # And the builder, because it is a module the mesh assigns rather than a program somebody # starts by hand — which is the only way its credential can be one the mesh delivered. - mesh-builder:development @@ -47,17 +39,10 @@ images: - mesh-provision-postgres:development # And the proxy, which is what turns a route grant into traffic actually arriving. - mesh-route-proxy:development - # And the cache, with its provisioner — the third provision after a database and a bucket, - # and the first whose tenancy is a keyspace rather than a namespace something else enforces. - - redis:7-alpine - mesh-provision-redis:development # And the object store's provisioner, so the module describing it can be planned. Without it # that module still names an image nothing serves, and planning it is refused — correctly. - mesh-provision-objectstore:development - # And a forge, so one of the real module descriptions can be started rather than only planned. - # It is the first of them to run: it needs a database from another module, a credential it did - # not choose, and a connection string it could not have written itself. - - gitea/gitea:1.22 place: all: [host, runtime] diff --git a/scenarios/whole-mesh-ace.yml b/scenarios/whole-mesh-ace.yml index b8465da..6be3329 100644 --- a/scenarios/whole-mesh-ace.yml +++ b/scenarios/whole-mesh-ace.yml @@ -10,11 +10,11 @@ # the mesh confirms the paths exist and mounts them, but creates and chowns none of it. # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock -# The runtimes are built by scripts/build-module-runtime.sh (one per module); every server image must -# be in the local daemon to be stocked. The media/app images are pulled by their pinned digests and -# tagged :mesh so repositoryFor matches the module.json paths (postgres/redis/portainer/mssql reuse -# their existing local tags). The test loads each committed module.json from mesh-catalog and rewrites -# its image references to what this scenario's own registry serves by digest. +# The runtimes are built by scripts/build-module-runtime.sh (one per module) and must be in the local +# daemon, because nothing serves them and nothing can. Every media/app image is pulled from the +# internet by the node itself, over its uplink, by the digest its module.json already pins. The test +# loads each committed module.json from mesh-catalog and rewrites only OUR image references, to the +# ID the machine holds each one under. scenario: whole-mesh-ace segments: @@ -25,50 +25,52 @@ segments: machines: anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 4GiB cpus: 4 disk: 20GiB + # The substrate only, so the control plane's image only. The runtimes belong on the node that + # runs the modules, and a 20GiB disk has no room for them anyway. + images: [mesh-control:development] # The whole ace service set — 24 modules, ~50 containers, several heavy (Plex, Home Assistant, # Letta ~1.8GiB, Baserow ~1.5GiB, the UniFi controller's JVM, mssql ~2GiB). Sized past novox. ace: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow memory: 18GiB cpus: 8 disk: 120GiB + # Every runtime. Not mesh-control: the control plane runs on the anchor. + images: + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-mssql:development + - mesh-runtime-portainer:development + - mesh-runtime-sonarr:development + - mesh-runtime-radarr:development + - mesh-runtime-lidarr:development + - mesh-runtime-plex:development + - mesh-runtime-bazarr:development + - mesh-runtime-nzbget:development + - mesh-runtime-qbittorrent:development + - mesh-runtime-jackett:development + - mesh-runtime-ombi:development + - mesh-runtime-tautulli:development + - mesh-runtime-bookshelf:development + - mesh-runtime-home-assistant:development + - mesh-runtime-mosquitto:development + - mesh-runtime-influxdb:development + - mesh-runtime-grafana:development + - mesh-runtime-baserow:development + - mesh-runtime-letta:development + - mesh-runtime-nodered:development + - mesh-runtime-searxng:development + - mesh-runtime-unifi:development images: - # The first-node substrate. - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The module server images. Reused local tags where the exact version does not matter for a boot - # (postgres/redis/portainer/mssql); pinned-digest :mesh tags for the media/app images. - - redis:7-alpine - - lscr.io/linuxserver/sonarr:mesh - - lscr.io/linuxserver/radarr:mesh - - lscr.io/linuxserver/lidarr:mesh - - lscr.io/linuxserver/bazarr:mesh - - lscr.io/linuxserver/nzbget:mesh - - lscr.io/linuxserver/qbittorrent:mesh - - lscr.io/linuxserver/jackett:mesh - - lscr.io/linuxserver/ombi:mesh - - lscr.io/linuxserver/tautulli:mesh - - lscr.io/linuxserver/unifi-controller:mesh - - plexinc/pms-docker:mesh - - ghcr.io/pennydreadful/bookshelf:mesh - - ghcr.io/home-assistant/home-assistant:mesh - - eclipse-mosquitto:mesh - - influxdb:mesh - - grafana/grafana:mesh - - baserow/baserow:mesh - - letta/letta:mesh - - nodered/node-red:mesh - - searxng/searxng:mesh - - valkey/valkey:mesh - - portainer/portainer-ce:latest - - mcr.microsoft.com/mssql/server:2022-latest # The per-module runtimes (built by scripts/build-module-runtime.sh). - mesh-runtime-postgres:development - mesh-runtime-redis:development diff --git a/scenarios/whole-mesh-full.yml b/scenarios/whole-mesh-full.yml index 2845078..1c9df35 100644 --- a/scenarios/whole-mesh-full.yml +++ b/scenarios/whole-mesh-full.yml @@ -1,95 +1,184 @@ -# The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the -# whole-mesh rehearsal (novox/hq). Combines scenarios/whole-mesh-novox.yml and whole-mesh-ace.yml. +# The FULL mesh in its REAL production shape: two segments, one access point, one overlay. # -# anchor — substrate ONLY (store, broker, control). -# novox — the 17-module novox set (providers + web apps + route-proxy + mailu + firewall). -# ace — the 24-module ace set (media/home stack), its /services/media library pre-created. +# This is the first multi-segment whole-mesh bed. The earlier flat whole-mesh-full sat every node +# on one public segment with a SEPARATE `anchor` carrying the substrate. Production is not flat, and +# there is no separate anchor: `novox` IS the anchor. It sits on the routable `hosting` segment, +# runs the substrate (store, broker, control) AND its own service set AND is the overlay hub and the +# public ingress. `ace`, `shanks` and `g14` sit on the household `home` segment BEHIND a NAT gateway +# — the access point — reachable from the outside only through what they dial out to. # -# An overlay is placed across all three so cross-node `at` resolves. Each service node is -# self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and -# the shared broker/store on anchor — which is exactly what this stage proves converges for two -# independent node-plans at once on one substrate. +# hosting (public, routable) home (private, behind the access point) +# novox 192.0.2.20 ── anchor ace 10.99.1.10 home server, media/IoT set +# substrate + novox set shanks 10.99.1.20 workstation (light) +# overlay hub, ingress g14 10.99.1.30 workstation (light) +# +# The `home` gateway masquerades v4 outbound and forwards inbound (an ordinary household router). +# Home nodes reach novox's public 192.0.2.20 by dialling OUT through it: the substrate broker (5671), +# the mesh's own artifact store, and — the thing this bed exists to prove — the WireGuard overlay hub +# (51820/udp). The hub keepalive holds the NAT hole open so the tunnel, once formed, stays up. novox +# cannot initiate to a home node at all; every home↔novox path is either the overlay or a forwarded +# port. +# +# EVERY NODE HAS EGRESS, which is not a convenience. Third-party images — postgres, the whole Mailu +# stack, Plex, everything the modules actually run — are pulled from the internet, because that is +# where a real node gets them. The lab used to serve them from a registry it raised inside the +# scenario; no production mesh has one, so a bootstrap that could only work against it went green +# here and would have failed anywhere else. What is loaded onto a machine now is only what exists in +# no registry at all: the mesh's own images, named per machine below. +# +# Egress is a SECOND path, not a replacement for the topology. Each node still reaches the rest of +# the scenario through its declared gateway — that is where the overlay handshake has to survive a +# masquerade — and the uplink carries only what leaves the scenario entirely. +# +# THE UNPROVEN THING (what the flat beds never tested): does the overlay tunnel FORM across the +# access point — a home node dialling novox's public hub endpoint, the handshake completing through +# the gateway's masquerade? The driving test verifies the WireGuard handshake and cross-segment +# reachability over the overlay explicitly, and reports form-vs-break as its headline. +# +# Substrate-on-novox collides on two host ports the separate-anchor beds never hit: the substrate +# store binds 127.0.0.1:5432 and novox's postgres provider publishes 5432; the substrate broker binds +# 5671 + 127.0.0.1:5672 and novox's lavinmq provider publishes 5672. The driving test REMAPS those two +# provider host publishes off the substrate's ports (consumers reach the providers over the mesh +# network on the container port, so the host side is free to move). Reported as a topology finding. # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock -# The images are the UNION of the two per-server scenarios; every one is already built/pulled by the -# per-server bed prerequisites (scripts/build-module-runtime.sh, build-route-proxy-image.sh, the -# mailu/keycloak and media :mesh digest pulls). +# MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap MESH_LAB_CATALOG=.../mesh-catalog/modules +# +# GENESIS AND JOINING ARE TWO DIFFERENT ACTS, and this bed distinguishes them. novox is brought +# into existence by `mesh-bootstrap` — the same program a bare machine runs — and is afterwards a +# working mesh of one, with a registry and a control plane that is an ordinary module pinned to an +# image that registry serves. ace, shanks and g14 then JOIN it: host binary, token, enrol, run. No +# bootstrap, no substrate, no registry. novox is never enrolled twice, because the installer +# already did it. +# +# The images: are the UNION of the novox set (feat/novox-conversions @ 431310f: the slug + roundcube +# fixes, so only-office/de-spiegel/amqp-email-forwarder now resolve) and the ace media/home set, and +# every one of them must already be BUILT on the workstation — nothing can fetch them. scenario: whole-mesh-full segments: + # The routable segment. novox lives here, and the overlay hub endpoint is a public address here. hosting: kind: public cidr: [192.0.2.0/24] + # The household segment behind the access point. Its gateway is an ordinary home router: it + # masquerades v4 outbound, forwards inbound, and expires idle mappings after two minutes — which + # is exactly the NAT hole a WireGuard keepalive has to hold open. + home: + kind: private + cidr: [10.99.1.0/24] + gateway: + to: hosting + address: [192.0.2.50] # what the world sees the household as + nat: [v4] + forwardable: true + mapping_ttl: 120s + machines: - anchor: - at: { segment: hosting, address: [192.0.2.10] } - inbound: allow - memory: 4GiB - cpus: 4 - disk: 20GiB + # The anchor: substrate (store, broker, control) + the whole novox service set + overlay hub + + # public ingress. Bigger than the flat bed's novox, because it now carries the substrate too. novox: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow - memory: 16GiB - cpus: 6 - disk: 100GiB + memory: 24GiB + cpus: 8 + disk: 130GiB + # The proxy, and a runtime per module novox is assigned. step-ca, photos, invoicing, novox.be, + # only-office, de-spiegel, amqp-email-forwarder, registry, firewall and fail2ban are not here + # because they carry no runtime image of their own — what they run is third-party or is the + # node itself. + # + # **mesh-control is NOT here, and its absence is the point** (novox/hq ADR 0067). The anchor is + # brought into existence by the installer, and the installer carries the control plane's image + # inside itself — that is the whole reason a machine that can reach no registry can still raise + # a mesh. Handing it over from the workstation as well would mean the bed never found out + # whether the installer really carries it: the load would say "already held" and the fiction + # would be invisible, which is exactly the class of thing the lab's own registry used to hide. + images: + - mesh-route-proxy:development + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-minio:development + - mesh-runtime-mongodb:development + - mesh-runtime-mssql:development + - mesh-runtime-lavinmq:development + - mesh-runtime-keycloak:development + - mesh-runtime-gitea:development + - mesh-runtime-nextcloud:development + - mesh-runtime-umami:development + - mesh-runtime-verdaccio:development + - mesh-runtime-portainer:development + - mesh-runtime-mailu:development + + # The home server: the whole ace media/home set — 24 modules, ~50 containers, several heavy + # (Plex, Home Assistant, Letta, Baserow, the UniFi JVM, mssql). Behind the gateway. ace: - at: { segment: hosting, address: [192.0.2.30] } + at: { segment: home, address: [10.99.1.10] } + egress: true inbound: allow memory: 18GiB cpus: 6 disk: 120GiB + # A runtime per module ace is assigned, and nothing of novox's. This is the point of saying it + # per machine: ace has no business holding a Keycloak runtime, and an operator's home server + # would not. + images: + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-mssql:development + - mesh-runtime-portainer:development + - mesh-runtime-sonarr:development + - mesh-runtime-radarr:development + - mesh-runtime-lidarr:development + - mesh-runtime-plex:development + - mesh-runtime-bazarr:development + - mesh-runtime-nzbget:development + - mesh-runtime-qbittorrent:development + - mesh-runtime-jackett:development + - mesh-runtime-ombi:development + - mesh-runtime-tautulli:development + - mesh-runtime-bookshelf:development + - mesh-runtime-home-assistant:development + - mesh-runtime-mosquitto:development + - mesh-runtime-influxdb:development + - mesh-runtime-grafana:development + - mesh-runtime-baserow:development + - mesh-runtime-letta:development + - mesh-runtime-nodered:development + - mesh-runtime-searxng:development + - mesh-runtime-unifi:development + # Two workstations on the same home LAN. Light on purpose: they enrol, join the overlay, and run + # one small module (portainer) so a real module converges on each without heavy load. Same-LAN + # nodes with no overlay endpoint of their own hairpin the hub rather than peering directly, which + # is the normal case and is fine. + shanks: + at: { segment: home, address: [10.99.1.20] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 30GiB + # One module, one runtime. Handing these two the whole union would put roughly thirty gigabytes + # of images onto a thirty-gigabyte disk, which is how you learn that "the lab loads everything + # everywhere" was never a description of anything real. + images: [mesh-runtime-portainer:development] + g14: + at: { segment: home, address: [10.99.1.30] } + egress: true + inbound: allow + memory: 3GiB + cpus: 2 + disk: 30GiB + images: [mesh-runtime-portainer:development] + +# The union of what the machines above ask for. This is the list the workstation must be able to +# export — every entry is one of the mesh's own images, built from source and published nowhere, so +# a machine holds it because it was handed it. Everything else the modules run comes from the +# internet and is not named here at all. images: - # --- substrate + shared --- - - postgres:17-alpine - - cloudamqp/lavinmq:latest - - mesh-control:development - - redis:7-alpine - - portainer/portainer-ce:latest - - mcr.microsoft.com/mssql/server:2022-latest - # --- novox server images --- - - minio/minio:latest - - mongo:7 - - quay.io/keycloak/keycloak:mesh - - gitea/gitea:1.22 - - nextcloud:stable - - ghcr.io/umami-software/umami:postgresql-latest - - alpine:latest - - verdaccio/verdaccio:6 - - registry:2 - - registry-api.novox.be/novox/invoicing-app:latest - - registry-api.novox.be/novox/invoicing-api:latest - - ghcr.io/mailu/unbound:mesh - - ghcr.io/mailu/admin:mesh - - ghcr.io/mailu/dovecot:mesh - - ghcr.io/mailu/postfix:mesh - - ghcr.io/mailu/rspamd:mesh - - ghcr.io/mailu/webmail:mesh - - ghcr.io/mailu/nginx:mesh - # --- ace server images --- - - lscr.io/linuxserver/sonarr:mesh - - lscr.io/linuxserver/radarr:mesh - - lscr.io/linuxserver/lidarr:mesh - - lscr.io/linuxserver/bazarr:mesh - - lscr.io/linuxserver/nzbget:mesh - - lscr.io/linuxserver/qbittorrent:mesh - - lscr.io/linuxserver/jackett:mesh - - lscr.io/linuxserver/ombi:mesh - - lscr.io/linuxserver/tautulli:mesh - - lscr.io/linuxserver/unifi-controller:mesh - - plexinc/pms-docker:mesh - - ghcr.io/pennydreadful/bookshelf:mesh - - ghcr.io/home-assistant/home-assistant:mesh - - eclipse-mosquitto:mesh - - influxdb:mesh - - grafana/grafana:mesh - - baserow/baserow:mesh - - letta/letta:mesh - - nodered/node-red:mesh - - searxng/searxng:mesh - - valkey/valkey:mesh # --- per-module runtimes (union) --- - mesh-runtime-postgres:development - mesh-runtime-redis:development @@ -97,11 +186,11 @@ images: - mesh-runtime-portainer:development - mesh-runtime-minio:development - mesh-runtime-mongodb:development + - mesh-runtime-lavinmq:development - mesh-runtime-keycloak:development - mesh-runtime-gitea:development - mesh-runtime-nextcloud:development - mesh-runtime-umami:development - - mesh-runtime-photos:development - mesh-runtime-verdaccio:development - mesh-runtime-mailu:development - mesh-route-proxy:development diff --git a/scenarios/whole-mesh-novox.yml b/scenarios/whole-mesh-novox.yml index e4cdcca..45ef3d5 100644 --- a/scenarios/whole-mesh-novox.yml +++ b/scenarios/whole-mesh-novox.yml @@ -17,9 +17,10 @@ # # MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock # The runtimes are built by scripts/build-module-runtime.sh (one per module that has code) and the -# route-proxy image by scripts/build-route-proxy-image.sh; every server image must be in the local -# daemon to be stocked. The test loads each committed module.json from mesh-catalog and rewrites its -# image references to what this scenario's own registry serves by digest. +# route-proxy image by scripts/build-route-proxy-image.sh; those must be in the local daemon, +# because nothing serves them and nothing can. Every third-party image is pulled from the internet +# over each node's uplink. The test loads each committed module.json from mesh-catalog and rewrites +# only OUR image references, to the ID the machine holds each one under. scenario: whole-mesh-novox segments: @@ -31,55 +32,48 @@ machines: # The substrate ONLY: store, broker, control. Nothing else lands here. anchor: at: { segment: hosting, address: [192.0.2.10] } + egress: true inbound: allow memory: 4GiB cpus: 4 disk: 20GiB + # The substrate only, so the control plane's image only. Handing this machine the whole set of + # runtimes would fill a 20GiB disk with images nothing on it will ever start. + images: [mesh-control:development] # The whole novox service set — ~38 containers (five providers with runtimes, six consumers with # runtimes, portainer/verdaccio/registry/route-proxy, the nine-container Mailu stack and its # runtime) plus two node-level modules. mssql alone wants ~2GiB; Mailu, Nextcloud and Keycloak are # each heavy. Sized well past the two-node-db bed's second node. novox: at: { segment: hosting, address: [192.0.2.20] } + egress: true inbound: allow memory: 16GiB cpus: 8 - # ~14GiB of images are pulled from the scenario's own registry by digest, several of them large + # ~14GiB of images are pulled from the internet over the uplink, several of them large # (mssql 1.7GiB, invoicing-api 1.9GiB, nextcloud 1.5GiB, umami/mongo ~0.9GiB), plus writable # layers and the runtimes. A hundred gigabytes holds the whole set without exhausting the disk # mid-apply. disk: 100GiB + # Every runtime, and the proxy. Not mesh-control: the control plane runs on the anchor. + images: + - mesh-runtime-postgres:development + - mesh-runtime-redis:development + - mesh-runtime-minio:development + - mesh-runtime-mongodb:development + - mesh-runtime-mssql:development + - mesh-runtime-keycloak:development + - mesh-runtime-gitea:development + - mesh-runtime-nextcloud:development + - mesh-runtime-umami:development + - mesh-runtime-photos:development + - mesh-runtime-portainer:development + - mesh-runtime-verdaccio:development + - mesh-runtime-mailu:development + - mesh-route-proxy:development images: - # The first-node substrate: store, broker, control. postgres:17-alpine doubles as the postgres - # provider's own service image (and Mailu's internal admin DB). - - postgres:17-alpine - - cloudamqp/lavinmq:latest - mesh-control:development - # The module server images. Each is stocked under the repository path its module.json names, so the - # test's rewrite (pinned(repositoryFor(image))) finds it. - - redis:7-alpine - - minio/minio:latest - - mongo:7 - - mcr.microsoft.com/mssql/server:2022-latest - - quay.io/keycloak/keycloak:mesh - - gitea/gitea:1.22 - - nextcloud:stable - - ghcr.io/umami-software/umami:postgresql-latest - - alpine:latest - - portainer/portainer-ce:latest - - verdaccio/verdaccio:6 - - registry:2 - - registry-api.novox.be/novox/invoicing-app:latest - - registry-api.novox.be/novox/invoicing-api:latest - # The Mailu stack (pulled by digest, tagged :mesh so repositoryFor matches the module.json paths). - - ghcr.io/mailu/unbound:mesh - - ghcr.io/mailu/admin:mesh - - ghcr.io/mailu/dovecot:mesh - - ghcr.io/mailu/postfix:mesh - - ghcr.io/mailu/rspamd:mesh - - ghcr.io/mailu/webmail:mesh - - ghcr.io/mailu/nginx:mesh # The per-module runtimes (built by scripts/build-module-runtime.sh). registry, route-proxy, # invoicing, firewall and fail2ban carry no mesh-runtime image; route-proxy ships its own. - mesh-runtime-postgres:development diff --git a/src/cli.ts b/src/cli.ts index e353173..5b624ee 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -233,10 +233,12 @@ async function main(): Promise { const seconds = ((Date.now() - started) / 1000).toFixed(1); console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`); if (raised.images.length > 0) { - // Printed because this is what a declaration pins, and it is not knowable until the - // scenario has been raised — the digest belongs to this registry. - console.log(`\nimages served, pinned by digest:`); - for (const image of raised.images) console.log(` ${image}`); + // Printed because this is what a declaration names them by, and it is not knowable until + // the image has been built — an image ID is the digest of its own configuration. + console.log(`\nthe mesh's own images, as the machines now hold them:`); + for (const image of raised.images) { + console.log(` ${image.requested} → ${image.reference}`); + } } if (scenario.snapshot) { const took = await snapshot(raised.instanceId, scenario.snapshot); diff --git a/src/declaration/parse.ts b/src/declaration/parse.ts index 38c490b..d1bd5a2 100644 --- a/src/declaration/parse.ts +++ b/src/declaration/parse.ts @@ -41,6 +41,9 @@ function normaliseMachine(raw: unknown): Machine { if (machine["memory"] !== undefined) result.memory = String(machine["memory"]); if (machine["cpus"] !== undefined) result.cpus = Number(machine["cpus"]); if (machine["disk"] !== undefined) result.disk = String(machine["disk"]); + // Absent and empty are different: absent means "all of the scenario's images", an explicit empty + // list means "none". A machine that runs nothing of ours should be able to say so. + if (machine["images"] !== undefined) result.images = toList(machine["images"]); return result; } diff --git a/src/declaration/types.ts b/src/declaration/types.ts index 343ef28..ae71d9f 100644 --- a/src/declaration/types.ts +++ b/src/declaration/types.ts @@ -110,6 +110,19 @@ export interface Machine { */ memory?: string; cpus?: number; + /** + * Which of the scenario's `images:` this machine is handed. + * + * Absent means all of them, which is right for a one-machine bed and wrong for a mesh: a + * workstation running one small module does not want forty runtimes copied onto a 30GiB disk. + * That is not a lab economy, it is what is true — an operator's machine holds the images its own + * modules need, because somebody put them there. + * + * Every entry must appear in the scenario's `images:`. Naming one that does not is refused + * rather than ignored, because a machine silently missing an image fails much later, inside an + * apply, as a container that will not start. + */ + images?: string[]; /** * Root disk size, e.g. "60GiB". Left unset, the VM uses the storage pool's default, which is * enough for a handful of modules. A broad install that stocks many runtime + service images @@ -142,11 +155,19 @@ export interface Scenario { policy?: Policy[]; place?: Placement; /** - * Container images this scenario needs inside it. + * **The mesh's own images** — the ones that exist in no registry and are put onto a machine by + * whoever built them. * - * A sealed machine cannot reach a registry, so the lab raises one on a public segment and - * serves these from it. Written as tags — the digest a declaration pins is the one THIS - * registry assigns, and it is reported when the scenario is raised. + * mesh-control, mesh-builder, mesh-route-proxy, the per-module runtimes and the provisioners are + * built from source and published nowhere. A machine gets them the way an operator's machine + * does: they are built on the workstation, loaded onto the machine, and named by the digest of + * their own image configuration. Written as tags, because a tag is what `docker save` can + * export; what a declaration then pins is the image ID, reported when the scenario is raised. + * + * **Third-party images do not belong here.** postgres, gitea, the mailu stack and everything + * else are pulled from the internet over a machine's `egress` uplink, exactly as they are in + * production. The lab used to serve them from a registry of its own, and that registry did not + * exist anywhere else — so every bootstrap problem it papered over went unfound. */ images?: string[]; /** Name the state once placement finishes, so a run can return to it. */ diff --git a/src/declaration/validate.ts b/src/declaration/validate.ts index ee6f2da..80df7f0 100644 --- a/src/declaration/validate.ts +++ b/src/declaration/validate.ts @@ -15,6 +15,7 @@ import type { Scenario, Segment } from "./types.ts"; import { contains, familyOf, parseAddress, parseCidr, type Cidr } from "./net.ts"; +import { mustBeHandedOver } from "../pinning.ts"; /** RFC 5737 and RFC 3849. The only addresses guaranteed never to route on the real internet. */ const DOCUMENTATION_RANGES = [ @@ -315,27 +316,36 @@ export function validate(scenario: Scenario): void { } } - for (const image of scenario.images ?? []) { + const declaredImages = scenario.images ?? []; + for (const image of declaredImages) { if (!image.trim()) { problems.push("images: an empty entry names nothing"); } else if (image.includes("@sha256:")) { - // The digest a declaration pins is the one the LAB's registry assigns, which is not - // knowable before the scenario is raised. Naming an upstream digest here would pin - // something this registry will never serve. + // A tag, because a tag is what `docker save` exports. The reference a declaration ends up + // using is the image's own ID, which is not knowable until the image has been built. problems.push( - `images: '${image}' is pinned by digest. Name it by tag — the lab's registry assigns ` + - `its own digest and reports it when the scenario is raised`, + `images: '${image}' is pinned by digest. Name it by tag — what a declaration uses is the ` + + `ID of the image loaded onto the machine, reported when the scenario is raised`, + ); + } else if (!mustBeHandedOver(image)) { + // **The rule that replaced the lab's registry.** Anything with somewhere to be fetched from + // is fetched from there, over the machine's uplink, exactly as in production. Serving it + // from inside the scenario instead is what hid the bootstrap faults this lab exists to find. + problems.push( + `images: '${image}' is not one of the mesh's own images, so nothing loads it. It is ` + + `pulled from the internet by the machine that needs it — give that machine 'egress: true' ` + + `and delete this line. Only mesh-* images, which exist in no registry, are placed by hand`, ); } } - if ((scenario.images ?? []).length > 0) { - const hasPublicV4 = Object.values(scenario.segments) - .some((s) => s.kind === "public" && s.cidr.some((c) => !c.includes(":"))); - if (!hasPublicV4) { - problems.push( - "images: this scenario declares images and has no public IPv4 segment to serve them " + - "from. The registry stands in for the outside world, so it sits on a public segment", - ); + for (const [name, machine] of Object.entries(scenario.machines)) { + for (const image of machine.images ?? []) { + if (!declaredImages.includes(image)) { + problems.push( + `machines.${name}.images: '${image}' is not in this scenario's images:. A machine can ` + + `only be handed one of the images the scenario says it has`, + ); + } } } diff --git a/src/lifecycle/address.ts b/src/lifecycle/address.ts index 6d6ed48..3bb9ed0 100644 --- a/src/lifecycle/address.ts +++ b/src/lifecycle/address.ts @@ -180,6 +180,20 @@ function withPrefix(scenario: Scenario, segment: string, address: string): strin * * The router's inside address is the first host address of the range, chosen rather than * declared because a scenario has nothing to say about it. + * + * **A machine with `egress` is routed differently, and it has to be.** The scenery inside a + * scenario — a gateway container, the transit router — reaches the scenario and nothing else: it + * has no route to the real internet, and never will, because it exists to reproduce a household + * router rather than to be one. So a default route pointing at it is a black hole for anything + * outside, and it wins over the uplink's DHCP route on metric. A machine that must pull an image + * would then sit there failing, with a default route that looks perfectly reasonable. + * + * So an egress machine keeps the uplink as its default and gets an EXPLICIT route to every other + * segment in the scenario, through the same gateway or transit it would otherwise have defaulted + * to. Where there is no such path, the range is made `unreachable` rather than left to fall + * through: 192.168.1.0/24 in a scenario is a documentation range in spirit but an ordinary private + * one in fact, and letting it escape to the uplink would put scenario traffic on whatever network + * the workstation happens to sit on. */ export async function applyDefaultRoutes( scenario: Scenario, @@ -195,6 +209,13 @@ export async function applyDefaultRoutes( // segment routes through transit instead — otherwise it can reach its own network and // nothing else, which is not what being on the internet means. const behind = spec.at.find((a) => scenario.segments[a.segment]?.gateway); + + if (spec.egress) { + await routeScenarioExplicitly(scenario, machine, name); + log(` routed ${machine} inside the scenario, its default out through the uplink`); + continue; + } + if (!behind) { await routeViaTransit(scenario, spec, name); continue; @@ -219,6 +240,104 @@ export async function applyDefaultRoutes( } } +/** One route a machine with egress needs, so the scenario stays reachable and stays inside. */ +export interface ScenarioRoute { + /** The range this route is for. */ + cidr: string; + /** The next hop inside the scenario, or null when there is none and the range is unreachable. */ + via: string | null; +} + +/** + * The routes a machine with egress needs into the rest of the scenario. + * + * Pure, and exported, because this is the decision that keeps the uplink and the declared gateway + * from fighting — and a decision only a full raise could check is one nobody checks. + * + * One route per segment the machine is not already on, through whatever it would have defaulted to: + * its gateway if it sits behind one, transit if it sits on a public segment and transit exists. + * What has no such path is `unreachable` — the faithful translation of the state it was in before, + * where its default route pointed into scenery that dropped it, and safer, because an unreachable + * route cannot be answered by whatever network the workstation happens to sit on. + */ +export function scenarioRoutesFor(scenario: Scenario, machine: string): ScenarioRoute[] { + const spec = scenario.machines[machine]; + if (!spec || spec.at === "detached") return []; + const at = spec.at; + const onSegments = new Set(at.map((a) => a.segment)); + const behindGateway = at.some((a) => scenario.segments[a.segment]?.gateway); + + const routes: ScenarioRoute[] = []; + for (const [segment, segmentSpec] of Object.entries(scenario.segments)) { + if (onSegments.has(segment)) continue; + for (const cidr of segmentSpec.cidr) { + const slash = cidr.lastIndexOf("/"); + if (slash === -1) continue; + const v6 = cidr.slice(0, slash).includes(":"); + routes.push({ + cidr, + via: behindGateway ? gatewayInside(scenario, at, v6) : transitOn(scenario, at, v6), + }); + } + } + return routes; +} + +/** + * Apply those routes, and leave the default to the uplink. + * + * No `dev`: every next hop here is on-link, so the kernel picks the interface, and asking `awk` to + * count links is one more thing that can pick `docker0`. + */ +async function routeScenarioExplicitly( + scenario: Scenario, + machine: string, + name: string, +): Promise { + for (const { cidr, via } of scenarioRoutesFor(scenario, machine)) { + const family = cidr.slice(0, cidr.lastIndexOf("/")).includes(":") ? "-6" : "-4"; + const route = via ? `${cidr} via ${via}` : `unreachable ${cidr}`; + await incus( + ["exec", name, "--", "sh", "-c", `ip ${family} route replace ${route} 2>/dev/null || true`], + 30_000, + ); + } +} + +/** The inside address of the gateway this machine sits behind: the first host address. */ +function gatewayInside(scenario: Scenario, at: Attachment[], v6: boolean): string | null { + const behind = at.find((a) => scenario.segments[a.segment]?.gateway); + if (!behind) return null; + for (const range of scenario.segments[behind.segment]?.cidr ?? []) { + const slash = range.lastIndexOf("/"); + if (slash === -1) continue; + const base = range.slice(0, slash); + if (base.includes(":") !== v6) continue; + if (v6) return `${base.replace(/::$/, "")}::1`; + const octets = base.split("."); + octets[3] = "1"; + return octets.join("."); + } + return null; +} + +/** The transit router's address on the public segment this machine sits on. */ +function transitOn(scenario: Scenario, at: Attachment[], v6: boolean): string | null { + // One public segment means everything public is adjacent and no transit router is raised, so + // there is nothing to point at — see raiseTransit. + const publicSegments = Object.values(scenario.segments).filter((s) => s.kind === "public"); + if (publicSegments.length < 2) return null; + + const onPublic = at.find((a) => scenario.segments[a.segment]?.kind === "public"); + if (!onPublic) return null; + for (const cidr of scenario.segments[onPublic.segment]?.cidr ?? []) { + if (cidr.slice(0, cidr.lastIndexOf("/")).includes(":") !== v6) continue; + const via = transitAddress(cidr); + if (via) return via.slice(0, via.lastIndexOf("/")); + } + return null; +} + /** A machine on a public segment reaches the other public networks through transit. */ async function routeViaTransit( scenario: Scenario, diff --git a/src/lifecycle/base.ts b/src/lifecycle/base.ts index b4f169d..a9ccdd7 100644 --- a/src/lifecycle/base.ts +++ b/src/lifecycle/base.ts @@ -87,10 +87,16 @@ export async function buildBaseImage( // Trust the documentation ranges as plain-HTTP registries. // - // A scenario's registry is scenery inside the scenario, serving over HTTP, and a runtime - // will not pull from one without being told. Scoped to RFC 5737 and RFC 3849 ranges rather - // than a specific address, because those never route on the real internet — so this cannot - // make a real machine trust a real registry, whatever it is copied onto. + // **Kept after the lab's own registry was deleted, because it was never only for that.** The + // mesh HAS a registry — the `registry` module, `mesh-registry`, serving artifacts to the whole + // mesh on port 5000 over plain HTTP from whatever node runs it. In a scenario that node's + // address is a documentation-range address, and a runtime will not pull from a plain-HTTP + // registry without being told to. Take this away and the artifact store is unusable from every + // machine but the one hosting it. + // + // Scoped to RFC 5737 and RFC 3849 ranges rather than a specific address, because those never + // route on the real internet — so this cannot make a real machine trust a real registry, + // whatever it is copied onto. await incus([ "exec", BUILDER, "--", "sh", "-c", `mkdir -p /etc/docker && printf '%s' '${JSON.stringify({ @@ -162,8 +168,8 @@ export async function buildBaseImage( // Read back that the runtime will actually pull over plain HTTP from a documentation // range. Writing the file is not the same as the daemon honouring it, and a base image - // that looks right here fails much later — in a sealed scenario, as a container that - // cannot fetch its image, which is a long way from the cause. + // that looks right here fails much later — as a container that cannot fetch its image + // from the mesh's own artifact store, which is a long way from the cause. const trusted = await incusOk( ["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"], 60_000, @@ -172,7 +178,8 @@ export async function buildBaseImage( throw new BaseImageError( `the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` + `registries. It reported: ${trusted?.trim() || "nothing"}\n` + - ` Every scenario raised from this image would fail to pull from its own registry.`, + ` Every scenario raised from this image would fail to pull from the mesh's own ` + + `artifact store, which serves plain HTTP inside the scenario.`, ); } log(" trusts the documentation ranges as registries"); diff --git a/src/lifecycle/egress.ts b/src/lifecycle/egress.ts new file mode 100644 index 0000000..ae1fd1b --- /dev/null +++ b/src/lifecycle/egress.ts @@ -0,0 +1,126 @@ +/** + * Confirming a machine that says it can reach the outside actually can. + * + * **This is what the registry-reachability check became.** The old one proved that every machine + * could fetch a manifest from the registry the lab raised inside the scenario — a real check of a + * fake path, since no production mesh has such a registry. What a machine actually does is pull + * from the internet, and that is now the thing worth proving before a raise says it is finished. + * + * The failure it exists to stop is the same one, in the same shape: `raise` returns, the caller + * applies a substrate, the first pull fails, no node enrols, and the instance is left a bare + * shell — with the cause several steps back and looking like a mesh fault rather than a lab one. + * + * Two things are checked, in this order, because they fail differently and the difference is the + * whole diagnosis: + * + * - **A name resolves.** Without this the machine has a route and no way to use it, and every + * pull dies inside the runtime saying it cannot look up a host. + * - **The path carries.** A request to the registry every image ultimately comes from, over the + * uplink, through whatever gateway sits in front of this machine. Any HTTP answer counts: what + * is in question is the path, not whether Docker Hub likes us. + */ + +import type { Scenario } from "../declaration/types.ts"; +import { incus, incusOk } from "../incus/client.ts"; + +export class EgressError extends Error { + constructor(message: string) { + super(message); + this.name = "EgressError"; + } +} + +/** The host every image is fetched through, in the end. Asked for, never pulled from, here. */ +const UPSTREAM = "registry-1.docker.io"; + +/** + * Confirm every machine declaring `egress` can resolve and reach the outside. + * + * Run after the routes and the firewalls, because that is the path a pull will take: a home node's + * default is the uplink, its route to the rest of the scenario is through its gateway, and its own + * filtering is in place. Checking earlier would prove something no pull relies on. + */ +export async function confirmEgress( + scenario: Scenario, + machineNames: Map, + log: (message: string) => void = () => {}, + waitSeconds = 120, +): Promise { + for (const [machine, spec] of Object.entries(scenario.machines)) { + if (!spec.egress || spec.at === "detached") continue; + const name = machineNames.get(machine); + if (!name) continue; + + if (!(await resolves(name, waitSeconds))) { + // One repair, then a verdict. The uplink is the lab's own network and its DHCP server is + // also its resolver, so the machine has been told the answer and may simply have nowhere + // to write it — an image without systemd-resolved leaves `UseDNS=yes` inert. + await pointResolverAtTheUplink(name); + if (!(await resolves(name, 30))) { + throw new EgressError( + `${machine} declares egress and cannot resolve ${UPSTREAM}.\n` + + ` It has a route out and no way to use it, so every image pulled from the internet ` + + `would fail inside the runtime as a lookup error.\n` + + ` The uplink's DHCP server is also its resolver; this machine has not taken it.`, + ); + } + } + + const code = await reaches(name, waitSeconds); + if (!code) { + throw new EgressError( + `${machine} declares egress, resolves names, and cannot reach ${UPSTREAM}.\n` + + ` This is the PATH: its default route, the uplink, or the host's own forwarding. ` + + `Every third-party image this machine needs is pulled from the internet, so anything ` + + `applied to it would stop at the first container.`, + ); + } + log(` ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`); + } +} + +async function resolves(name: string, waitSeconds: number): Promise { + const deadline = Date.now() + waitSeconds * 1_000; + while (Date.now() < deadline) { + const said = await incusOk( + ["exec", name, "--", "sh", "-c", `getent hosts ${UPSTREAM} >/dev/null && echo yes`], 30_000, + ); + if (said?.trim() === "yes") return true; + await new Promise((r) => setTimeout(r, 3_000)); + } + return false; +} + +/** + * Any HTTP status at all, which is what "the path carries" means. + * + * Not 200: an unauthenticated `/v2/` is answered 401 by design, and a check demanding 200 would + * fail on a machine whose network is perfect. + */ +async function reaches(name: string, waitSeconds: number): Promise { + const deadline = Date.now() + waitSeconds * 1_000; + while (Date.now() < deadline) { + const said = (await incusOk( + ["exec", name, "--", "sh", "-c", + `curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${UPSTREAM}/v2/`], 40_000, + ))?.trim(); + if (said && /^[1-5][0-9]{2}$/.test(said)) return said; + await new Promise((r) => setTimeout(r, 5_000)); + } + return null; +} + +/** + * Write a resolver of last resort: the uplink's own gateway, which serves DHCP and DNS both. + * + * Deliberately the machine's default next hop rather than a name looked up somewhere — for a + * machine with egress that is the uplink by construction, since every scenario range is routed + * explicitly and nothing else defaults. + */ +async function pointResolverAtTheUplink(name: string): Promise { + await incus([ + "exec", name, "--", "sh", "-c", + `via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` + + `[ -n "$via" ] && printf 'nameserver %s\\n' "$via" > /etc/resolv.conf; true`, + ], 30_000); +} diff --git a/src/lifecycle/operate.ts b/src/lifecycle/operate.ts index 9dc8165..e85a712 100644 --- a/src/lifecycle/operate.ts +++ b/src/lifecycle/operate.ts @@ -61,11 +61,45 @@ export async function exec( */ timeoutMs = 120_000, ): Promise<{ stdout: string; stderr: string }> { + const name = await instanceNameOf(instanceId, machine); + return incus(["exec", name, "--", ...command], timeoutMs); +} + +/** + * What the hypervisor calls one of a scenario's machines. + * + * Asked of the daemon by the metadata each instance carries, and only derived from the naming + * rule when it answers nothing — the same order `exec` has always used. It is exported because + * the placement stage takes this name rather than the pair, and a caller that wants to put + * something on one machine after the raise (the installer, a catalogue checkout) would otherwise + * have to reimplement the lookup and get the fallback wrong. + */ +export async function instanceNameOf(instanceId: string, machine: string): Promise { const found = (await taggedInstances()).find( (i) => i.instanceId === instanceId && i.machine === machine, ); - const name = found?.name ?? machineName(instanceId, machine); - return incus(["exec", name, "--", ...command], timeoutMs); + return found?.name ?? machineName(instanceId, machine); +} + +/** + * Put a file from this workstation inside a machine. + * + * The long default timeout is not generosity: what goes through here is an installer carrying a + * container image, which is tens of megabytes, and a push that is merely slow must not look like + * a push that is stuck. + */ +export async function push( + instanceId: string, + machine: string, + local: string, + remote: string, + mode?: string, + timeoutMs = 900_000, +): Promise { + const name = await instanceNameOf(instanceId, machine); + const args = ["file", "push", local, `${name}${remote}`]; + if (mode) args.push("--mode", mode); + await incus(args, timeoutMs); } /** diff --git a/src/lifecycle/place.ts b/src/lifecycle/place.ts index eb245a6..be85aec 100644 --- a/src/lifecycle/place.ts +++ b/src/lifecycle/place.ts @@ -19,6 +19,7 @@ import { join } from "node:path"; import { incus, incusOk, succeeds } from "../incus/client.ts"; import { around, log, shorten } from "../log.ts"; +import { mustBeHandedOver, repositoryOf, type HeldImage } from "../pinning.ts"; /** * What this stage can put inside a machine. @@ -41,6 +42,17 @@ export function isPlaceable(artifact: string): boolean { /** Where the host binary lives on a machine once placed. */ export const HOST_PATH = "/usr/local/bin/mesh-host"; +/** + * Where the installer lives on a machine once placed. + * + * **Beside the host, because it is the same tier and the same delivery** (novox/hq ADR 0067): + * bootstrapping is done by hand and it changes a machine, which is what tier 0 is — but `mesh-host` + * says of itself that it connects to nothing and listens on nothing, and an installer that loads + * images and interrogates a control plane cannot be folded into it without making that sentence + * false. Two programs, one shelf. + */ +export const BOOTSTRAP_PATH = "/usr/local/bin/mesh-bootstrap"; + export interface Placement { machine: string; artifacts: string[]; @@ -77,6 +89,14 @@ export function hostBinaryPath(): string | null { return process.env["MESH_LAB_HOST_BINARY"] ?? null; } +/** + * The installer to place, from the environment. Same rule as the host binary: an explicit path, + * and nothing is guessed. + */ +export function bootstrapBinaryPath(): string | null { + return process.env["MESH_LAB_BOOTSTRAP_BINARY"] ?? null; +} + export class PlacementError extends Error { readonly machine: string; @@ -145,6 +165,41 @@ export async function placeHost( return { machine, profile, version }; } +/** + * Put the installer on a machine and ask it what it is. + * + * The same shape as {@link placeHost} and for the same reason: the version is read back from the + * running binary, because a file arriving is not a program working (novox/hq ADR 0018). The + * installer is the larger of the two by an order of magnitude — it carries a saved container image + * — so a copy that half-arrived is a real possibility rather than a theoretical one. + * + * It is placed on ONE machine, not all of them. Only the anchor is bootstrapped; every other + * machine joins a mesh that already exists, with the host binary and a token and nothing else. + */ +export async function placeBootstrap( + instanceName: string, + machine: string, + binary: string, + log: (message: string) => void = () => {}, +): Promise { + await incus(["file", "push", binary, `${instanceName}${BOOTSTRAP_PATH}`, "--mode", "0755"], + 900_000); + + const version = (await incusOk( + ["exec", instanceName, "--", BOOTSTRAP_PATH, "version"], 60_000, + ))?.trim(); + if (!version) { + throw new PlacementError( + machine, + `the installer was copied to ${machine} and does not run there. It carries a saved ` + + `container image and is twenty megabytes or so, which is exactly the size at which a ` + + `truncated copy stops being theoretical.`, + ); + } + log(` placed the installer on ${machine} (${version})`); + return version; +} + /** Place everything a scenario declares. */ export async function applyPlacements( scenario: Scenario, @@ -263,17 +318,17 @@ export async function placeImage( // sealed machine cannot reach. Measured, not assumed: the load says `Loaded image ID:` // instead of `Loaded image:`, and `docker images` then lists nothing. // - // This collides with novox/hq ADR 0006, which pins bundle images BY DIGEST and has the host - // refuse anything else. Reconciling the two needs a registry inside the scenario, which is - // real design work — see 04-ISSUES/009. + // What an archive DOES keep is the image's own ID — the digest of its configuration — and that + // is how the mesh's own images are named once loaded. See {@link loadHeldImages}. if (reference.includes("@sha256:")) { throw new PlacementError( machine, `${reference} is pinned by digest, and an image placed from an archive cannot keep its ` + `digest — a repo digest only exists for an image a registry served.\n` + ` Placing it would load an image with no name, and a container declaring that digest ` + - `would try to reach a registry the machine cannot see.\n` + - ` Place it by tag, or give the scenario a registry (novox/hq 04-ISSUES/009).`, + `would try to reach a registry.\n` + + ` Place it by tag. What survives being loaded is the image's own ID, which is what a ` + + `declaration names it by (mesh-host: an image the machine already holds).`, ); } @@ -397,3 +452,153 @@ async function waitForRuntime(instanceName: string, machine: string): Promise ({ machine, images: spec.images ?? all })) + .filter((plan) => plan.images.length > 0); +} + +/** + * Put the mesh's own images onto the machines that need them, and say what they are now called. + * + * **This is what replaced the lab's registry**, and the difference is the whole point. A registry + * inside the scenario served every image — third-party ones included — from an address that exists + * in no production mesh, so a bootstrap that could only work against it went green here and would + * have failed anywhere else. There is no such registry now: third-party images are pulled from the + * internet over each machine's `egress` uplink, and the mesh's own arrive the way they arrive on an + * operator's machine — somebody built them and put them there. + * + * The reference a declaration then uses is the image's **own ID**, the digest of its configuration. + * `docker load` preserves it, so the name is identical on the workstation that built the image and + * on every machine handed a copy — immutable, unforgeable, and requiring nothing to have served it + * (mesh-host, *an image may be named by the digest of its own configuration*). + * + * Read back on both sides. The ID is taken from the workstation and then CONFIRMED on the machine, + * because a load that lands a different image than the one exported is exactly the silent fault + * this lab exists to catch — and the reference is what every manifest will be rewritten to. + */ +export async function loadHeldImages( + scenario: Scenario, + machineNames: Map, + log: (message: string) => void = () => {}, +): Promise { + const plans = planHeldImages(scenario); + if (plans.length === 0) return []; + + const held: HeldImage[] = []; + for (const requested of scenario.images ?? []) { + // Refused by the validator, so reaching here would be a validator bug — but the consequence + // is a third-party image quietly loaded from the workstation instead of pulled, which is the + // fiction all of this exists to remove. Cheap to check, expensive to miss. + if (!mustBeHandedOver(requested)) { + throw new Error( + `images: '${requested}' is not one of the mesh's own images. It is pulled from the ` + + `internet by the machine that needs it, not loaded from this workstation.`, + ); + } + + const wanted = plans.filter((plan) => plan.images.includes(requested)).map((p) => p.machine); + if (wanted.length === 0) continue; + + const onThisWorkstation = (await local( + "docker", ["image", "inspect", "--format", "{{.Id}}", requested], 60_000, + )).stdout.trim(); + if (!/^sha256:[0-9a-f]{64}$/.test(onThisWorkstation)) { + throw new Error( + `${requested} is not on this workstation, so there is nothing to hand the machines.\n` + + ` It is one of the mesh's own images and exists in no registry — nothing can pull it.\n` + + ` Build it first (mesh-control's \`make image …\`, or scripts/build-module-runtime.sh).`, + ); + } + + // **An image id belongs to the runtime holding it, and does not survive the journey.** It is + // the digest of the image's *configuration*, and a runtime rewrites that configuration as it + // loads: this workstation saves in one format and the machine's older runtime stores it in + // another, so the same bytes arrive under a different name. Measured, not assumed — the same + // image was b86bb81c… here and 2dc21904… on the machine. + // + // So the id is READ BACK from the machine rather than predicted from here. Predicting it is + // what the earlier version did, and it failed at the only useful moment: the manifests would + // have been rewritten to a reference no machine holds, and nothing serves these images, so + // every apply would have stopped at the container with a pull that cannot succeed. + let reference = ""; + + // Exported once, handed to each machine that asked for it. The archive is the expensive part + // and it does not depend on the destination. + const tar = join(tmpdir(), `mesh-lab-held-${process.pid}-${Date.now()}.tar`); + const saved = await local("docker", ["save", requested, "-o", tar], 900_000); + if (!saved.ok) { + await unlink(tar).catch(() => {}); + throw new Error(`cannot export ${requested} from this workstation: ${saved.stderr.trim()}`); + } + + try { + for (const machine of wanted) { + const name = machineNames.get(machine); + if (!name) continue; + await waitForRuntime(name, machine); + await incus(["file", "push", tar, `${name}/tmp/held.tar`], 900_000); + const loaded = await incusOk( + ["exec", name, "--", "docker", "load", "-i", "/tmp/held.tar"], 900_000, + ); + if (!loaded?.includes("Loaded image")) { + throw new PlacementError( + machine, + `${requested} was pushed to ${machine} and did not load.\n` + + ` The runtime said: ${loaded?.trim() || "nothing"}`, + ); + } + // What this machine calls it, asked of the tag it was just loaded under. This is the + // reference every manifest naming this image will be rewritten to. + const there = (await incusOk( + ["exec", name, "--", "docker", "image", "inspect", "--format", "{{.Id}}", requested], + 120_000, + ))?.trim() ?? ""; + if (!/^sha256:[0-9a-f]{64}$/.test(there)) { + throw new PlacementError( + machine, + `${requested} loaded onto ${machine} and the runtime will not say what it holds.\n` + + ` It answered '${there || "nothing"}'.\n` + + ` Nothing serves this image, so a manifest naming it has only what the machine ` + + `itself reports — and there is nothing to fall back to.`, + ); + } + // **A manifest carries one reference, so the machines must agree on it.** They are built + // from one base image and load one archive, so they do; if that ever stops being true the + // image cannot be named at all from a catalogue, and that is worth stopping for rather + // than rewriting to whichever machine answered last. + if (!reference) { + reference = there; + } else if (reference !== there) { + throw new PlacementError( + machine, + `${requested} is ${there} on ${machine} and ${reference} on a machine already ` + + `loaded.\n` + + ` One manifest cannot name both, and this image exists in no registry to be ` + + `named by instead. The machines' runtimes differ in a way that changes how they ` + + `store what they are given.`, + ); + } + await succeeds(["exec", name, "--", "rm", "-f", "/tmp/held.tar"], 60_000); + } + } finally { + await unlink(tar).catch(() => {}); + } + + held.push({ requested, repository: repositoryOf(requested), reference }); + log(` ${requested} → ${reference.slice(0, 19)}… on ${wanted.join(", ")}`); + } + return held; +} diff --git a/src/lifecycle/raise.ts b/src/lifecycle/raise.ts index 2891dd0..0c1aa9c 100644 --- a/src/lifecycle/raise.ts +++ b/src/lifecycle/raise.ts @@ -22,9 +22,10 @@ import { applyAddresses, applyDefaultRoutes } from "./address.ts"; import { assertSupported } from "./supported.ts"; import { planRouters, raiseRouters, raiseTransit } from "./router.ts"; import { applyHostFirewalls } from "./firewall.ts"; -import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts"; +import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements, loadHeldImages } from "./place.ts"; import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts"; -import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts"; +import { confirmEgress } from "./egress.ts"; +import type { HeldImage } from "../pinning.ts"; import { log as record } from "../log.ts"; /** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */ @@ -47,12 +48,15 @@ export interface RaisedScenario { networks: string[]; pool: string; /** - * Images the scenario's registry serves, as references a declaration can pin. + * The mesh's own images, as loaded onto the machines, and what a declaration should call them. * - * Reported rather than declared, because the digest is the one this registry assigned and - * is not knowable before it was raised. + * Reported rather than declared: an image built from source has no digest until it has been + * built, and what names it here is the digest of its own configuration. + * + * **Only ours.** Everything third-party is pulled from the internet by the machine that needs + * it, so it is not in this list and nothing rewrites it. */ - images: string[]; + images: HeldImage[]; } export class RaiseError extends Error { @@ -314,24 +318,6 @@ export async function raise( const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log); if (transit) routers.push(transit); - // Stocked on this workstation, where there is a network, and served from inside the - // scenario, where there is not (novox/hq 04-ISSUES/009). - enter("stocking the registry"); - const stock = await stockRegistry(scenario.images ?? [], log); - let registry: Awaited> = null; - try { - enter("raising the registry"); - registry = await raiseRegistry(scenario, instanceId, stock, log); - } finally { - // Cleaning up scratch must not fail a raise that succeeded. The scenario is standing - // and usable; a directory left behind is untidy, and saying so is the honest report. - try { - await discardStock(stock); - } catch (err) { - log(` (could not remove the registry's scratch directory: ${(err as Error).message})`); - } - } - enter("routing machines through their gateways"); await applyDefaultRoutes(scenario, byMachine, log); @@ -340,16 +326,29 @@ export async function raise( enter("applying host firewalls"); await applyHostFirewalls(scenario, byMachine, log); + // **Only now is "this machine can reach the outside" a true statement.** The route, the + // gateway and the machine's own filtering are all in place, so this is the path a pull takes. + // A raise that returned without checking would hand the next step a fact it depends on and + // has no way to test — which is how a substrate apply used to die on its first pull. + enter("confirming egress reaches the internet"); + await confirmEgress(scenario, byMachine, log); + // Last, and only once the underlay is real. Placing before the machines can reach each // other would test the host against a network the scenario does not describe. enter("placing"); await applyPlacements(scenario, byMachine, log); + // After `placing`, because loading an image needs the container runtime that `placing` + // confirmed. The mesh's own images only — everything third-party is pulled by the machine + // itself, over its uplink, exactly as it is on a real one. + enter("loading the mesh's own images onto the machines"); + const images = await loadHeldImages(scenario, byMachine, log); + return { instanceId, scenario: scenario.scenario, - images: registry?.pinned ?? [], - machines: [...created, ...routers, ...(registry ? [registry.machine] : [])], + images, + machines: [...created, ...routers], networks, pool, }; diff --git a/src/lifecycle/registry.ts b/src/lifecycle/registry.ts deleted file mode 100644 index 4daefd8..0000000 --- a/src/lifecycle/registry.ts +++ /dev/null @@ -1,405 +0,0 @@ -/** - * A registry inside the scenario. - * - * A sealed machine cannot reach a registry, and an image placed from an archive cannot keep its - * digest — `docker save` of a digest reference produces an archive with no repo tag, because a - * repo digest only exists for an image a registry served (novox/hq 04-ISSUES/009). So an image - * pinned by digest, which is the only kind the host accepts - * ([ADR 0006](../../02-DECISIONS/0046-the-installer-fetches-what-it-pins.md)), could not be - * placed at all. - * - * The answer is a registry, and it is not a workaround for the lab: ADR 0006 names an OCI - * registry as substrate, and ADR 0006 says a first node fetches "upstream, wherever the image - * ordinarily lives". **This is that upstream** — scenery, like the transit router is the - * internet ([ADR 0016](../../02-DECISIONS/0033-a-router-is-scenery-not-a-node.md)). - * - * The digests it serves are its own, not Docker Hub's, and that is correct rather than a - * compromise. What ADR 0006 requires is a reference that is exact and cannot move. A digest - * assigned by this registry is both. - */ - -import { spawn } from "node:child_process"; - -import { incus, incusOk, succeeds } from "../incus/client.ts"; -import { macFor, networkName } from "./names.ts"; -import { addressLink } from "./address.ts"; -import { around, log, shorten } from "../log.ts"; -import { BASE_IMAGE_ALIAS, placeImage } from "./place.ts"; -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -/** The image the registry itself runs from. Placed by tag, which archives keep. */ -export const REGISTRY_IMAGE = "registry:2"; - -/** Where the registry serves, inside its machine. */ -export const REGISTRY_PORT = 5000; - -export class RegistryError extends Error { - constructor(message: string) { - super(message); - this.name = "RegistryError"; - } -} - -export interface StockedImage { - /** What the scenario asked for, as written. */ - requested: string; - /** The repository path the registry serves it under. */ - repository: string; - /** The digest THIS registry assigned. What a declaration pins. */ - digest: string; -} - -export interface Stock { - /** A directory holding the registry's data, ready to be placed in a machine. */ - dataDir: string; - images: StockedImage[]; -} - -/** - * Build a registry's data directory on this workstation, with the given images in it. - * - * Runs a throwaway registry here — where there IS a network — pushes into it, and keeps what - * it wrote. Research 012's reframing again: fetch at build time on a machine that has a - * network, apply on a target that needs nothing. - * - * The caller owns the returned directory and must remove it. - */ -export async function stockRegistry( - references: string[], - log: (message: string) => void = () => {}, -): Promise { - if (references.length === 0) return { dataDir: "", images: [] }; - - const dataDir = await mkdtemp(join(tmpdir(), "mesh-lab-registry-")); - const container = `mesh-lab-stock-${process.pid}`; - const port = 5000 + (process.pid % 1000); - - await docker(["rm", "-f", container], 60_000); - const started = await docker( - ["run", "-d", "--name", container, "-p", `${port}:5000`, "-v", `${dataDir}:/var/lib/registry`, - REGISTRY_IMAGE], - 300_000, - ); - if (!started.ok) { - await rm(dataDir, { recursive: true, force: true }); - throw new RegistryError( - `cannot run ${REGISTRY_IMAGE} on this workstation to stock a registry: ${started.stderr.trim()}`, - ); - } - - try { - await waitForRegistry(port); - const images: StockedImage[] = []; - - for (const reference of references) { - // The repository path a machine will pull from. A tag is dropped: what a declaration - // pins is the digest, and carrying the tag as well would invite pinning the wrong one. - const repository = repositoryFor(reference); - const target = `localhost:${port}/${repository}`; - - const tagged = await docker(["tag", reference, target], 60_000); - if (!tagged.ok) { - throw new RegistryError( - `${reference} is not on this workstation, and the lab does not fetch on a scenario's ` + - `behalf. Pull it here first.\n ${tagged.stderr.trim()}`, - ); - } - const pushed = await docker(["push", target], 900_000); - if (!pushed.ok) throw new RegistryError(`cannot push ${reference}: ${pushed.stderr.trim()}`); - - const digest = digestFrom(pushed.stdout + pushed.stderr); - if (!digest) { - throw new RegistryError( - `${reference} was pushed and the registry did not report a digest. Without one there ` + - `is nothing for a declaration to pin.`, - ); - } - images.push({ requested: reference, repository, digest }); - log(` stocked ${repository}@${digest}`); - } - - return { dataDir, images }; - } catch (err) { - await discardStock({ dataDir, images: [] }); - throw err; - } finally { - await docker(["rm", "-f", container], 60_000); - } -} - -/** - * Remove a stocked registry's data. - * - * Through a container, because a container wrote it. The registry runs as root inside, so the - * blobs it writes into a bind mount are owned by root and an ordinary process cannot remove - * them — `rmdir` fails with EACCES on a directory that looks like ours. - * - * Whoever made the files removes them. - */ -export async function discardStock(stock: Stock): Promise { - if (!stock.dataDir) return; - await docker(["run", "--rm", "-v", `${stock.dataDir}:/stock`, REGISTRY_IMAGE, - "sh", "-c", "rm -rf /stock/* /stock/.[!.]* 2>/dev/null || true"], 120_000); - await rm(stock.dataDir, { recursive: true, force: true }).catch(() => {}); -} - -/** `alpine:3.20` and `alpine` both serve from `alpine`; `foo/bar:1` from `foo/bar`. */ -export function repositoryFor(reference: string): string { - const withoutDigest = reference.split("@")[0] ?? reference; - const lastColon = withoutDigest.lastIndexOf(":"); - const lastSlash = withoutDigest.lastIndexOf("/"); - return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; -} - -/** `docker push` prints `: digest: sha256:… size: …` on its last useful line. */ -export function digestFrom(output: string): string | null { - const match = output.match(/digest:\s*(sha256:[a-f0-9]{64})/); - return match?.[1] ?? null; -} - -async function waitForRegistry(port: number): Promise { - for (let i = 0; i < 30; i++) { - const probe = await docker(["run", "--rm", "--network", "host", REGISTRY_IMAGE, - "sh", "-c", `wget -q -O- http://localhost:${port}/v2/ >/dev/null 2>&1`], 30_000); - if (probe.ok) return; - await new Promise((r) => setTimeout(r, 1_000)); - } - throw new RegistryError("a registry was started on this workstation and never answered"); -} - -function docker( - args: string[], - timeoutMs: number, -): Promise<{ ok: boolean; stdout: string; stderr: string }> { - // The second of the three places the lab runs an external program (novox/hq 04-ISSUES/024). - // `docker push` of a large image is minutes of legitimate silence, which is exactly when a - // heartbeat earns its keep. - return around(`docker ${shorten(args)}`, () => runDocker(args, timeoutMs), { heartbeatMs: 15_000 }); -} - -function runDocker( - args: string[], - timeoutMs: number, -): Promise<{ ok: boolean; stdout: string; stderr: string }> { - return new Promise((resolve) => { - const child = spawn("docker", args, { stdio: ["ignore", "pipe", "pipe"] }); - let stdout = ""; - let stderr = ""; - const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs); - child.stdout.on("data", (d) => (stdout += d)); - child.stderr.on("data", (d) => (stderr += d)); - child.on("error", (err) => { - clearTimeout(timer); - resolve({ ok: false, stdout, stderr: err.message }); - }); - child.on("close", (code) => { - clearTimeout(timer); - // A docker failure is an answer here rather than an exception, so it would otherwise pass - // through the log looking exactly like a success. - if (code !== 0) { - log.debug(` exit ${code}: ${shorten([stderr.trim() || "(nothing on stderr)"], 400)}`); - } - resolve({ ok: code === 0, stdout, stderr }); - }); - }); -} - -// --- the registry inside a scenario ------------------------------------------------------------ - -/** - * Where the registry sits on its segment. - * - * A convention rather than a declaration, like the router's. `.250` is chosen to sit well away - * from the low addresses scenarios give their machines, so a scenario can be written without - * thinking about it and a collision is obvious when it happens. - */ -export const REGISTRY_HOST_OCTET = 250; - -/** The address the registry answers on, given the segment it is attached to. */ -export function registryAddress(cidr: string): string { - const [network] = cidr.split("/"); - const parts = (network ?? "").split("."); - if (parts.length !== 4) { - throw new RegistryError( - `cannot place a registry on '${cidr}': it is not an IPv4 network, and the registry needs ` + - `an address a machine can be pointed at.`, - ); - } - return `${parts[0]}.${parts[1]}.${parts[2]}.${REGISTRY_HOST_OCTET}`; -} - -/** What a declaration should pin, once a scenario is raised. */ -export function pinnedReference(address: string, image: StockedImage): string { - return `${address}:${REGISTRY_PORT}/${image.repository}@${image.digest}`; -} - -// --- raising it inside a scenario --------------------------------------------------------------- - -/** What a raised registry is, and what a declaration needs from it. */ -export interface RaisedRegistry { - machine: string; - segment: string; - address: string; - /** Each image, as a reference a declaration can pin. */ - pinned: string[]; -} - -/** - * Pick the segment the registry sits on. - * - * A public segment, because that is what stands in for the outside world — a first node fetches - * from upstream, and this is upstream. An IPv4 range, because a machine has to be pointed at it - * by address. - */ -export function registrySegment( - segments: Record, -): { name: string; cidr: string } | null { - for (const [name, segment] of Object.entries(segments)) { - if (segment.kind !== "public") continue; - const v4 = segment.cidr.find((c) => !c.includes(":")); - if (v4) return { name, cidr: v4 }; - } - return null; -} - -/** - * Raise a registry inside the scenario and load the stocked images into it. - * - * Scenery, in the same sense the transit router is: nothing under test runs on it, it holds no - * identity, and no assertion is made about its internals. It exists so that a machine can fetch - * an image the way a real one does — over the network, from a registry, by digest. - */ -export async function raiseRegistry( - scenario: { segments: Record }, - instanceId: string, - stock: Stock, - log: (message: string) => void = () => {}, -): Promise { - if (stock.images.length === 0) return null; - - const segment = registrySegment(scenario.segments); - if (!segment) { - throw new RegistryError( - `this scenario declares images and has no public IPv4 segment to serve them from.\n` + - ` The registry stands in for the outside world, so it sits on a public segment.`, - ); - } - - const address = registryAddress(segment.cidr); - const name = `mlab-${instanceId}-registry`; - const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/")); - - if (!(await succeeds(["config", "show", name], 15_000))) { - await incus([ - "init", BASE_IMAGE_ALIAS, name, "--vm", - "-c", "security.secureboot=false", - "-c", "limits.memory=1GiB", - "-c", `user.mesh-lab.instance=${instanceId}`, - // Tagged as a machine as well, so `destroy` finds it with one query — a router that - // carried only its own tag was left behind and held its networks open. - "-c", "user.mesh-lab.machine=registry", - "-c", "user.mesh-lab.registry=true", - ], 300_000); - await succeeds(["config", "device", "remove", name, "eth0"], 15_000); - await incus([ - "config", "device", "add", name, "eth0", "nic", - "nictype=bridged", - `parent=${networkName(instanceId, segment.name)}`, - `hwaddr=${macFor(instanceId, "registry", 0)}`, - ]); - } - await succeeds(["start", name], 60_000); - await waitForAgent(name); - - // Addressed the way every other machine is: a systemd-networkd unit matching the MAC. - // - // **This used to be `ip addr add`, and it stalled the lab.** An address set by hand leaves - // networkd waiting to configure a link it was never told about, so the link sits at - // `configuring`, `systemd-networkd-wait-online` never returns — its timeout is `infinity` — - // and `network-online.target` is never reached. Docker is ordered after that target, so - // `docker load` two lines below blocked on a socket whose daemon was queued behind a target - // that would never come. - // - // Matching on MAC and not on interface name is still the rule: a machine with a container - // runtime has a `docker0` that sorts before `enp5s0`, and naive selection configures that. - await addressLink(name, { - device: "eth0", - mac: macFor(instanceId, "registry", 0), - addresses: [`${address}${prefix}`], - // The registry takes the segment's default. It carried no MTU before this and still does - // not: what a scenario sets an MTU for is the path under test, and this is scenery. - mtu: undefined, - }); - - log(` registry on ${segment.name} at ${address}`); - - // The registry's own image, placed by tag — an archive keeps a tag and cannot keep a digest, - // which is the whole reason this machine exists. - // Logged, not silenced. This is the step a stall sat in for thirty-five minutes while the - // caller had passed it a callback that threw everything away (novox/hq 04-ISSUES/024). - await placeImage(name, "registry", REGISTRY_IMAGE, log); - - // The destination must EXIST before a recursive push, or incus copies the source's contents - // rather than the source — the data lands one directory too shallow, the registry finds - // nothing where it looks, and every pull fails with `not found`. - await incus(["exec", name, "--", "mkdir", "-p", "/srv/registry"], 60_000); - await incus(["file", "push", "-r", `${stock.dataDir}/docker`, `${name}/srv/registry/`], 900_000); - - await incus(["exec", name, "--", "docker", "run", "-d", - "--name", "registry", "--restart", "unless-stopped", - "-p", `${REGISTRY_PORT}:5000`, - "-v", "/srv/registry:/var/lib/registry", - REGISTRY_IMAGE], 300_000); - - // Read back that each image is SERVED, by asking for its manifest by digest — which is - // exactly what a machine will do. - // - // Not that the catalog endpoint answers: `{"repositories":[]}` contains the word - // `repositories`, so checking for that passed on a registry holding nothing at all, and the - // failure surfaced much later as a container that could not be pulled. - let answered = false; - for (let i = 0; i < 20 && !answered; i++) { - const ping = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null", - "-w", "%{http_code}", "--max-time", "3", - `http://localhost:${REGISTRY_PORT}/v2/`], 30_000); - answered = ping?.trim() === "200"; - if (!answered) await new Promise((r) => setTimeout(r, 2_000)); - } - if (!answered) { - throw new RegistryError( - `the registry on ${name} started and never answered. Machines in this scenario cannot ` + - `fetch an image, so nothing that declares a container will work.`, - ); - } - - const pinned: string[] = []; - for (const image of stock.images) { - const code = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null", - "-w", "%{http_code}", "--max-time", "5", - "-H", "Accept: application/vnd.docker.distribution.manifest.v2+json", - `http://localhost:${REGISTRY_PORT}/v2/${image.repository}/manifests/${image.digest}`, - ], 60_000); - if (code?.trim() !== "200") { - throw new RegistryError( - `the registry on ${name} is running and does not serve ${image.repository}@${image.digest} ` + - `(it answered ${code?.trim() || "nothing"}).\n` + - ` The images were stocked on this workstation and did not arrive intact, so a ` + - `machine declaring that image would fail to pull it.`, - ); - } - const reference = pinnedReference(address, image); - pinned.push(reference); - log(` serving ${reference}`); - } - return { machine: name, segment: segment.name, address, pinned }; -} - -async function waitForAgent(name: string): Promise { - for (let i = 0; i < 90; i++) { - if (await succeeds(["exec", name, "--", "true"], 10_000)) return; - await new Promise((r) => setTimeout(r, 2_000)); - } - throw new RegistryError(`${name} started and its agent never answered.`); -} diff --git a/src/pinning.ts b/src/pinning.ts index 110f2e1..3197cfd 100644 --- a/src/pinning.ts +++ b/src/pinning.ts @@ -1,55 +1,141 @@ /** - * Rewriting an image reference to the one a scenario's own registry serves. + * Naming the mesh's own images by what they are. * * **A digest is not knowable until something is built** (novox/hq 04-ISSUES/025). A manifest in a * repository can pin a third-party image, because somebody can ask a registry what a tag points - * at. It cannot pin an image the mesh builds itself: that image does not exist yet, and when it - * does its digest belongs to whichever registry served it. + * at. It cannot pin an image the mesh builds itself: mesh-control, mesh-builder, mesh-route-proxy, + * the per-module runtimes and the provisioners exist in no registry, so there is no manifest + * digest to write down. The catalogue ships sixty-four zeros for them, which parses, resolves, + * composes — and stops on the machine. * - * The bundle has always had this problem and solves it by rewriting references once the scenario's - * registry is up and its digests are known. Modules have exactly the same problem and were solving - * it by shipping sixty-four zeros, which parses, resolves, composes — and stops on the machine. + * The lab used to answer that with a registry of its own: raise one inside the scenario, push + * everything into it, and rewrite every reference — third-party ones included — to the digest it + * assigned. **That registry does not exist in production, so the lab was testing a fiction**, and + * the fiction hid the bootstrap problems it was supposed to find. * - * So the rewriting is shared rather than copied, and matches on the **repository**, because that - * is the part a person writes and the only part that survives being served somewhere else. + * What is true instead is two things: + * + * - **Third-party images are pulled from the internet.** They are left exactly as written, and + * the machine fetches them over its `egress` uplink the way any machine does. + * - **The mesh's own images are built and handed over.** They are loaded onto the machine from + * the workstation that built them, and named by the digest of their own image configuration — + * a bare `sha256:…`, which mesh-host accepts as "an image this machine already holds" + * (mesh-host, *an image may be named by the digest of its own configuration*). + * + * So the rewriting that remains is only the second kind, and it matches on the **repository**, + * because that is the part a person writes and the only part a placeholder digest does not say. */ -/** `192.0.2.250:5000/ghcr.io/mailu/admin@sha256:…` → `ghcr.io/mailu/admin` */ -export function repositoryOf(pinned: string): string { - const at = pinned.indexOf("@"); - const body = at === -1 ? pinned : pinned.slice(0, at); - const slash = body.indexOf("/"); - // Everything after the registry. A reference with no slash at all is its own repository. - return slash === -1 ? body : body.slice(slash + 1); +/** What the lab loaded onto a machine, and what a declaration should call it. */ +export interface HeldImage { + /** As the scenario asked for it, a tag this workstation holds: `mesh-runtime-postgres:development`. */ + requested: string; + /** What a manifest names it by, with no tag and no digest: `mesh-runtime-postgres`. */ + repository: string; + /** + * The reference a declaration uses: a bare `sha256:<64 hex>`. + * + * The image's own ID — the digest of its configuration — which `docker load` preserves, so the + * name is the same on the workstation that built it and on every machine it was handed to. + */ + reference: string; +} + +/** `alpine:3.20` and `alpine` both mean `alpine`; `foo/bar:1` means `foo/bar`. */ +export function repositoryOf(reference: string): string { + const withoutDigest = reference.split("@")[0] ?? reference; + const lastColon = withoutDigest.lastIndexOf(":"); + const lastSlash = withoutDigest.lastIndexOf("/"); + return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; } /** - * Replace every reference to a stocked repository with the reference this scenario serves. + * Whether a reference names an image the mesh builds for itself. * - * Matching is on the repository and ignores whatever registry and digest were written down — - * a file may name `postgres@sha256:7456…` or `mesh-provision-postgres@sha256:0000…` and both mean - * *the postgres this scenario has*. That is the whole point: the text says which image, the - * scenario says which copy. + * **Derived from the shape the build produces, not from a list of names.** `make image + * builder-image provisioner-image objectstore-image redis-provisioner-image proxy-image` in + * mesh-control and `scripts/build-module-runtime.sh` here both tag their output `mesh-` + * with no registry host and no upstream organisation — that is what "built here, published + * nowhere" looks like, and a hardcoded list would go stale the first time a module is added. * - * A repository the scenario did not stock is left alone rather than blanked. It may be reachable - * some other way, and silently emptying a reference would produce the exact failure this exists to - * prevent. + * The absence of a slash carries the weight: `ghcr.io/mailu/admin` and + * `registry.example/novox/www` both say where they are fetched from, and a bare + * `mesh-runtime-plex` says there is nowhere. */ -export function pinnedInto(text: string, served: string[]): string { +export function isMeshBuilt(reference: string): boolean { + const repository = repositoryOf(reference); + return !repository.includes("/") && repository.startsWith("mesh-"); +} + +/** + * Registries an anonymous pull works against. + * + * Not a list of what is trusted — a list of where no account is needed. Everything else wants one, + * and a scenario machine has none. + */ +const PUBLIC_REGISTRIES = [ + "docker.io", "ghcr.io", "quay.io", "lscr.io", "gcr.io", "registry.k8s.io", + "public.ecr.aws", "mcr.microsoft.com", "docker.elastic.co", "registry.gitlab.com", +]; + +/** The registry a reference names, or "" when it names none and so means Docker Hub. */ +export function registryOf(reference: string): string { + const first = repositoryOf(reference).split("/")[0] ?? ""; + // A first segment is a registry only if it looks like a host: `novox/www` is an organisation on + // Docker Hub; `registry.example/novox/www` is somewhere else entirely. + return first.includes(".") || first.includes(":") ? first : ""; +} + +/** + * Whether the workstation has to hand this image over rather than let the machine fetch it. + * + * **Two reasons, one consequence.** An image the mesh builds for itself exists in no registry at + * all. An image in the operator's *private* registry exists in one the machines have no account + * for, and the pull fails with `no basic auth credentials` — which is not something more patience + * fixes. Either way the machine cannot get it alone, so the workstation, which does hold the + * credential, exports it and loads it. + * + * **This stands in for something, and it is worth saying what.** In a finished mesh these are built + * by the mesh's builder and published to the mesh's own store, and every machine pulls them from + * there with a credential the mesh granted it. Until that store exists there is nowhere for them to + * come from — and handing them over is the closest honest thing to it, rather than a registry the + * lab invents, which is exactly what was just removed. + */ +export function mustBeHandedOver(reference: string): boolean { + if (isMeshBuilt(reference)) return true; + const registry = registryOf(reference); + return registry !== "" && !PUBLIC_REGISTRIES.includes(registry); +} + +/** + * Replace every reference to one of the mesh's own images with the image the machine holds. + * + * Matching is on the repository and ignores whatever digest was written down — a manifest says + * `mesh-runtime-postgres@sha256:0000…` and means *the runtime this machine was given*. + * + * **Everything else is left exactly as it is.** `postgres@sha256:7456…`, `gitea/gitea@sha256:…` + * and `ghcr.io/mailu/admin@sha256:…` are pulled from the internet over the machine's uplink, which + * is what a real machine does and the reason the lab's own registry is gone. + */ +export function pinnedInto(text: string, held: HeldImage[]): string { let out = text; - for (const pinned of served) { - const repository = repositoryOf(pinned); - const escaped = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + for (const image of held) { + const escaped = image.repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // Optionally a registry, then the repository, then any digest. Anchored on a quote or // whitespace so a longer repository ending in a shorter one is not half-replaced. out = out.replaceAll( new RegExp(`(?<=^|["\\s])(?:[A-Za-z0-9_.:-]+\\/)*${escaped}@sha256:[0-9a-f]{64}`, "g"), - pinned, + image.reference, ); } return out; } +/** The reference for one repository, or undefined if this scenario loaded no such image. */ +export function referenceFor(held: HeldImage[], repository: string): string | undefined { + return held.find((image) => image.repository === repository)?.reference; +} + /** Whether anything is still pinned to a placeholder, which would fail on the machine. */ export function stillUnpinned(text: string): string[] { return [...text.matchAll(/([A-Za-z0-9_.:/-]+)@sha256:0{64}/g)].map((m) => m[1]!); diff --git a/src/rebuild.ts b/src/rebuild.ts index c945348..d12336c 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -2,7 +2,8 @@ * Rebuild what the lab runs, from source, before it runs. * * **A stale artifact reporting success against old rules is the fault this project keeps writing - * down** (novox/hq 04-ISSUES/005). The lab consumes three artifacts from two repositories, and they + * down** (novox/hq 04-ISSUES/005). The lab consumes a handful of artifacts from two repositories, + * and they * were rebuilt by hand, one at a time, from memory. A rename in the control plane's catalogue needs * both the control-plane image *and* the builder binary, because both parse manifests; rebuilding * one left a binary eleven hours old refusing a field the mesh had just renamed, and cost a full @@ -73,9 +74,40 @@ export function planned(env: NodeJS.ProcessEnv = process.env): Build[] { }); } } + + // **The installer, carrying the control plane's image.** + // + // Last, and that is an ordering rather than a preference: `make bootstrap` embeds the output of + // `docker save `, so the image has to have been built by the step above or the installer + // carries whatever was lying around — the eleven-hour-old artifact again, this time inside a + // binary where nothing would ever notice. + // + // It is built here at all because the bed now bootstraps THROUGH it (novox/hq ADR 0067): the + // anchor is brought into existence by running the same program a bare machine runs, rather than + // by the bed applying a substrate bundle by hand and calling that an install. An installer that + // was stale would be a bed proving something about last week's procedure. + const installer = env["MESH_LAB_BOOTSTRAP_BINARY"]; + if (installer && where["mesh-host"]) { + builds.push({ + what: "installer", + in: where["mesh-host"], + argv: ["make", "bootstrap", `IMAGE=${controlPlaneImage(env)}`, `BOOTSTRAP_OUT=${installer}`], + }); + } return builds; } +/** + * The control-plane image the installer carries. + * + * `mesh-control:development` is what mesh-control's `make image` tags, and what the scenarios name + * — one tag, said in one place. It is overridable because a release installer carries a release + * image, and nothing about that is the lab's business. + */ +export function controlPlaneImage(env: NodeJS.ProcessEnv = process.env): string { + return env["MESH_LAB_CONTROL_IMAGE"] ?? "mesh-control:development"; +} + /** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */ export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] { const built: string[] = []; diff --git a/src/suite.ts b/src/suite.ts index 8496026..47682be 100644 --- a/src/suite.ts +++ b/src/suite.ts @@ -53,9 +53,9 @@ export async function runSuite(args: string[]): Promise { // the long run reaches the end of that path in about 160 seconds. // // So it cost a whole scenario, every passing run, to save about 45 seconds on a failing one. - // A scenario is three machines including a registry that boots a kernel to serve files, which - // is where the two minutes went. `test/integration/canary.test.ts` is still there and still - // runs when it is named; it is no longer raised on the way to everything else. + // A scenario is machines that each boot a kernel, which is where the two minutes went. + // `test/integration/canary.test.ts` is still there and still runs when it is named; it is no + // longer raised on the way to everything else. const { code, seen } = await runFiles(files); console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files, process.env, against))); diff --git a/src/warm.ts b/src/warm.ts index f8762d0..902a243 100644 --- a/src/warm.ts +++ b/src/warm.ts @@ -23,6 +23,7 @@ import { homedir } from "node:os"; import { list, restore, snapshot, snapshots, destroy } from "./lifecycle/operate.ts"; import type { Against } from "./lastrun.ts"; import { whatWasTested } from "./lastrun.ts"; +import type { HeldImage } from "./pinning.ts"; /** The state a warm instance is kept at. One label, because a second is a state nobody named. */ export const label = "warm"; @@ -31,13 +32,13 @@ export interface Warm { scenario: string; instanceId: string; /** - * The image references the scenario's registry serves, pinned by digest. + * The mesh's own images, as loaded onto this instance's machines, by the ID each is held under. * * Kept because they are worked out while raising and a restored instance never raises. Without - * them a warm run knows nothing about what it can pull, and every test naming an image fails - * for a reason that has nothing to do with what it was testing. + * them a warm run knows nothing about what its machines hold, and every test naming one of our + * images fails for a reason that has nothing to do with what it was testing. */ - images: string[]; + images: HeldImage[]; /** The commit each repository was at when this was brought to its state. */ against: Against; at: string; @@ -187,23 +188,23 @@ export async function cool(): Promise { } /** - * What a raised scenario stocked, held until it is kept. + * What a raised scenario loaded onto its machines, held until it is kept. * * Raising works the images out and snapshotting happens later, so this carries them between the * two without the caller having to hold them. */ -const stock = new Map(); +const stock = new Map(); -export function rememberStock(instanceId: string, images: string[]): void { +export function rememberStock(instanceId: string, images: HeldImage[]): void { stock.set(instanceId, images); } -function stockOf(instanceId: string): string[] { +function stockOf(instanceId: string): HeldImage[] { return stock.get(instanceId) ?? []; } -/** What a restored instance's registry serves, from when it was warmed. */ -export function warmStock(instanceId: string): { images: string[] } { +/** What a restored instance's machines hold, from when it was warmed. */ +export function warmStock(instanceId: string): { images: HeldImage[] } { const warm = remembered(); if (!warm || warm.instanceId !== instanceId) return { images: [] }; return { images: warm.images }; diff --git a/test/diagram.test.ts b/test/diagram.test.ts index 64c922a..6766642 100644 --- a/test/diagram.test.ts +++ b/test/diagram.test.ts @@ -109,7 +109,7 @@ test("segments are ordered public first, then by depth behind them", () => { }); test("a declared address appears on the machine that holds it", () => { - assert.match(xml, /192\.168\.1\.135/); + assert.match(xml, /10\.99\.1\.135/); assert.match(xml, /198\.51\.100\.7/); }); diff --git a/test/egress-routes.test.ts b/test/egress-routes.test.ts new file mode 100644 index 0000000..9e26a72 --- /dev/null +++ b/test/egress-routes.test.ts @@ -0,0 +1,128 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { parseScenario } from "../src/declaration/parse.ts"; +import { scenarioRoutesFor } from "../src/lifecycle/address.ts"; + +/** + * The uplink and the declared gateway must not fight. + * + * **This is the one decision the registry's removal turned on, and it is invisible in a raise.** + * Every machine that needs an image now has an `egress` uplink, and the uplink's DHCP offers a + * default route. So did the scenario: a machine behind a household gateway defaulted through it, a + * machine on a public segment defaulted through transit. Both of those are containers that reach + * the scenario and nothing else — no route to the real internet, by design, because they exist to + * reproduce a household router rather than to be one. + * + * A default route through either is therefore a black hole for anything outside, and it beats the + * uplink's route on metric. The machine would sit failing every pull with a routing table that + * looks perfectly reasonable. + * + * The answer is that an egress machine states the scenario's ranges explicitly and lets the uplink + * be the default. These tests are how that is checked without spending an hour raising four nodes. + */ + +const HOUSEHOLD = ` +scenario: household +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + home: + kind: private + cidr: [10.99.1.0/24] + gateway: + to: hosting + address: [192.0.2.50] + nat: [v4] + forwardable: true +machines: + novox: + at: { segment: hosting, address: [192.0.2.20] } + egress: true + ace: + at: { segment: home, address: [10.99.1.10] } + egress: true + sealed: + at: { segment: home, address: [10.99.1.99] } +`; + +test("a machine behind a gateway still reaches the scenario through that gateway", () => { + // The whole point of the topology: home→public is a masqueraded outbound path, and the overlay + // handshake has to survive it. An egress machine that stopped using its gateway would be + // testing a flat network with extra steps. + const routes = scenarioRoutesFor(parseScenario(HOUSEHOLD), "ace"); + assert.deepEqual(routes, [{ cidr: "192.0.2.0/24", via: "10.99.1.1" }]); +}); + +test("a machine's own segment gets no route — it is already on-link", () => { + const routes = scenarioRoutesFor(parseScenario(HOUSEHOLD), "ace"); + assert.ok(!routes.some((r) => r.cidr === "10.99.1.0/24"), JSON.stringify(routes)); +}); + +/** + * **The dangerous one.** `home` is 10.99.1.0/24 — a documentation range in spirit, an ordinary + * private one in fact, and very possibly the network the workstation itself is on. + * + * With one public segment there is no transit router, so novox has no path to `home` at all. Left + * to fall through, that traffic would leave by the uplink and land on whatever the workstation can + * reach. Unreachable is both the faithful reproduction of what it had before — a default route into + * scenery that dropped it — and the only safe answer. + */ +test("a range with no path inside the scenario is unreachable, not leaked to the uplink", () => { + const routes = scenarioRoutesFor(parseScenario(HOUSEHOLD), "novox"); + assert.deepEqual(routes, [{ cidr: "10.99.1.0/24", via: null }]); +}); + +test("a machine without egress is left to its default route, and states nothing", () => { + // Not because it needs no routes — it has one, a default through its gateway, applied the old + // way. This function is only asked about machines whose default belongs to the uplink. + const scenario = parseScenario(HOUSEHOLD); + assert.equal(scenario.machines["sealed"]?.egress, undefined); +}); + +const TWO_PUBLIC = ` +scenario: two-public +segments: + hosting: + kind: public + cidr: [192.0.2.0/24] + elsewhere: + kind: public + cidr: [198.51.100.0/24] +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10] } + egress: true +`; + +test("with a second public segment the transit router is the way across, as it always was", () => { + // Transit is raised only when there is more than one public segment, so this is exactly the + // case where pointing at it means something. + const routes = scenarioRoutesFor(parseScenario(TWO_PUBLIC), "anchor"); + assert.deepEqual(routes, [{ cidr: "198.51.100.0/24", via: "192.0.2.254" }]); +}); + +const V6 = ` +scenario: both-families +segments: + hosting: + kind: public + cidr: [192.0.2.0/24, "2001:db8:a::/48"] + elsewhere: + kind: public + cidr: [198.51.100.0/24, "2001:db8:b::/48"] +machines: + anchor: + at: { segment: hosting, address: [192.0.2.10, "2001:db8:a::10"] } + egress: true +`; + +test("each family is routed through its own next hop", () => { + // A v6 range routed via a v4 next hop is not a route, and the reverse is not either. + const routes = scenarioRoutesFor(parseScenario(V6), "anchor"); + assert.deepEqual(routes, [ + { cidr: "198.51.100.0/24", via: "192.0.2.254" }, + { cidr: "2001:db8:b::/48", via: "2001:db8:a::fffe" }, + ]); +}); diff --git a/test/integration/anthropic-bed.test.ts b/test/integration/anthropic-bed.test.ts index 4c85c42..85a4c9a 100644 --- a/test/integration/anthropic-bed.test.ts +++ b/test/integration/anthropic-bed.test.ts @@ -49,7 +49,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -73,7 +74,7 @@ const ACCESS_TOKEN = "at-lab-access-token-minted-by-the-stub"; const ROTATED_REFRESH = "rt-lab-rotated-still-only-the-manager"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -113,20 +114,13 @@ async function meshTry(command: string): Promise<{ out: string; ok: boolean }> { return on(`docker exec mesh-control /mesh-control ${command}`); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -177,7 +171,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/assigned-audit.test.ts b/test/integration/assigned-audit.test.ts index e94365c..9c4a412 100644 --- a/test/integration/assigned-audit.test.ts +++ b/test/integration/assigned-audit.test.ts @@ -7,7 +7,7 @@ * container that connects over amqps with that account — never the broker's own. The trail filling * is the proof the delivered, scoped credential authenticated and the subscription bound. * - * It needs the host binary, the substrate bundle, and the runtime image stocked by the scenario: + * It needs the host binary, the substrate bundle, and the runtime image the scenario loads: * * MESH_LAB_HOST_BINARY=.../mesh-host * MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock @@ -22,7 +22,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -40,8 +41,8 @@ const SCENARIO = "audit-node"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -67,22 +68,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -125,7 +119,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); @@ -151,7 +145,7 @@ after(async () => { test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", { skip, timeout: 900_000, }, async () => { - // The assigned-module manifest (mesh-catalog), its runtime image the digest this registry serves. + // The assigned-module manifest (mesh-catalog), its runtime image the ID the machine holds. const manifest = JSON.stringify({ module: "audit-logger", version: "1", diff --git a/test/integration/assigned-catalogue-apps.test.ts b/test/integration/assigned-catalogue-apps.test.ts index 428006c..aa583c6 100644 --- a/test/integration/assigned-catalogue-apps.test.ts +++ b/test/integration/assigned-catalogue-apps.test.ts @@ -30,7 +30,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -48,8 +49,8 @@ const SCENARIO = "catalogue-apps"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -75,22 +76,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -133,7 +127,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-catalogue-media.test.ts b/test/integration/assigned-catalogue-media.test.ts index 79bcb4b..c75fad6 100644 --- a/test/integration/assigned-catalogue-media.test.ts +++ b/test/integration/assigned-catalogue-media.test.ts @@ -26,7 +26,8 @@ * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock * scripts/build-module-runtime.sh {sonarr,radarr} build the runtime images into the local daemon; * scenarios/catalogue-media.yml stocks them. lscr.io/linuxserver/{sonarr,radarr} must be in the - * local daemon to be stocked. Each *arr runtime is given a lab API key so its client constructs and + * local daemon; the service images are pulled from the internet. Each *arr runtime is given a lab + * API key so its client constructs and * its tools register (as plex is given a lab token) — the server need not be configured by hand. */ @@ -37,7 +38,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -55,8 +57,8 @@ const SCENARIO = "catalogue-media"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -82,22 +84,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -140,7 +135,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-catalogue-mqtt.test.ts b/test/integration/assigned-catalogue-mqtt.test.ts index fb02003..5bd307d 100644 --- a/test/integration/assigned-catalogue-mqtt.test.ts +++ b/test/integration/assigned-catalogue-mqtt.test.ts @@ -26,7 +26,7 @@ * scripts/build-module-runtime.sh mosquitto builds mesh-runtime-mosquitto:development (carrying * mosquitto_ctrl and the compiled bootstrap entrypoint) into the local daemon, which * scenarios/catalogue-mqtt.yml stocks. eclipse-mosquitto:2 must be in the local daemon to be - * stocked; the host pulls both from the scenario's own registry by digest. + * the host pulls both from the internet over its uplink, by the digests the catalogue pins. */ import { test, before, after } from "node:test"; @@ -36,7 +36,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -54,7 +55,7 @@ const SCENARIO = "catalogue-mqtt"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -80,22 +81,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -138,7 +132,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-catalogue-small.test.ts b/test/integration/assigned-catalogue-small.test.ts index 2f7972f..e32c950 100644 --- a/test/integration/assigned-catalogue-small.test.ts +++ b/test/integration/assigned-catalogue-small.test.ts @@ -21,7 +21,7 @@ * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock * scripts/build-module-runtime.sh {postgres,redis,minio,plex} build the four runtime images into the * local daemon; scenarios/catalogue-small.yml stocks them. postgres:17-alpine, redis:7-alpine and - * minio/minio:latest must be in the local daemon to be stocked. + * minio/minio:latest is pulled from the internet by the node itself. */ import { test, before, after } from "node:test"; @@ -31,7 +31,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -49,8 +50,8 @@ const SCENARIO = "catalogue-small"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -76,22 +77,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -140,7 +134,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-grafana.test.ts b/test/integration/assigned-grafana.test.ts index 0b0b37f..bea54a3 100644 --- a/test/integration/assigned-grafana.test.ts +++ b/test/integration/assigned-grafana.test.ts @@ -22,7 +22,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -40,7 +41,7 @@ const SCENARIO = "grafana-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -65,20 +66,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -121,7 +115,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index 255182c..abdd1cf 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -38,7 +38,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -58,8 +59,8 @@ const SCENARIO = "model-usage-bed"; const NODE = "laptop"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -85,22 +86,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -164,7 +158,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/assigned-plex.test.ts b/test/integration/assigned-plex.test.ts index de5b638..33c4b5c 100644 --- a/test/integration/assigned-plex.test.ts +++ b/test/integration/assigned-plex.test.ts @@ -10,7 +10,7 @@ * proof the invocation routed to the assigned runtime, ran plex's real code, and replied, all under * the scoped account and never the broker's own. * - * It needs the host binary, the substrate bundle, and the runtime image stocked by the scenario: + * It needs the host binary, the substrate bundle, and the runtime image the scenario loads: * * MESH_LAB_HOST_BINARY=.../mesh-host * MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock @@ -25,7 +25,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -43,8 +44,8 @@ const SCENARIO = "plex-node"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -70,22 +71,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -128,7 +122,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-redis.test.ts b/test/integration/assigned-redis.test.ts index 574f031..647d889 100644 --- a/test/integration/assigned-redis.test.ts +++ b/test/integration/assigned-redis.test.ts @@ -12,7 +12,7 @@ * has no way yet to deliver one to a provider's runtime (04-ISSUES). The manifest here sets a * lab-local key so the mechanism can be proven; the delivery is a separate, open design question. * - * It needs the host binary, the substrate bundle, and the runtime image stocked by the scenario: + * It needs the host binary, the substrate bundle, and the runtime image the scenario loads: * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock * scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development into the local @@ -26,7 +26,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -44,7 +45,7 @@ const SCENARIO = "redis-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -69,20 +70,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -125,7 +119,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/assigned-schedule-tick.test.ts b/test/integration/assigned-schedule-tick.test.ts index dfceb1e..7cf53f8 100644 --- a/test/integration/assigned-schedule-tick.test.ts +++ b/test/integration/assigned-schedule-tick.test.ts @@ -38,7 +38,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -56,8 +57,8 @@ const SCENARIO = "schedule-tick"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -83,22 +84,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -155,7 +149,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-sonarr.test.ts b/test/integration/assigned-sonarr.test.ts index 1232d37..656f903 100644 --- a/test/integration/assigned-sonarr.test.ts +++ b/test/integration/assigned-sonarr.test.ts @@ -20,7 +20,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -38,7 +39,7 @@ const SCENARIO = "sonarr-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -63,20 +64,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -119,7 +113,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/assigned-tools-confluence.test.ts b/test/integration/assigned-tools-confluence.test.ts index 24d17c2..681ebf3 100644 --- a/test/integration/assigned-tools-confluence.test.ts +++ b/test/integration/assigned-tools-confluence.test.ts @@ -29,7 +29,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -47,8 +48,8 @@ const SCENARIO = "tools-confluence"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -74,22 +75,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -132,7 +126,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-tools-gitlab.test.ts b/test/integration/assigned-tools-gitlab.test.ts index 118178d..70fcc19 100644 --- a/test/integration/assigned-tools-gitlab.test.ts +++ b/test/integration/assigned-tools-gitlab.test.ts @@ -28,7 +28,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -46,8 +47,8 @@ const SCENARIO = "tools-gitlab"; const MACHINE = "anchor"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -73,22 +74,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -131,7 +125,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/assigned-two-node-db.test.ts b/test/integration/assigned-two-node-db.test.ts index 223108d..33ea3d0 100644 --- a/test/integration/assigned-two-node-db.test.ts +++ b/test/integration/assigned-two-node-db.test.ts @@ -44,7 +44,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -63,8 +64,8 @@ const SCENARIO = "two-node-db"; const NODE = "laptop"; let instanceId = ""; -/** What the scenario's registry serves, by digest. */ -let stocked: string[] = []; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -90,22 +91,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -173,7 +167,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // The first node raises the substrate — store, broker, control — from the bundle its host carries, // its digests rewritten to the ones this scenario's own registry serves. diff --git a/test/integration/builds.test.ts b/test/integration/builds.test.ts index 674645a..539068b 100644 --- a/test/integration/builds.test.ts +++ b/test/integration/builds.test.ts @@ -20,7 +20,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; @@ -42,7 +43,22 @@ const skip = !capability.usable const SCENARIO = "first-node"; const MACHINE = "anchor"; let instanceId = ""; -let registry = ""; + +/** + * Where a build publishes to. + * + * **The mesh has a registry, and this is that one.** `mesh-catalog/modules/registry` serves the + * mesh's artifact store on port 5000; a build publishes into it. This test starts the same image + * on the machine directly rather than assigning the module, because what is under test is the + * build chain and not module delivery. + * + * It used to publish into the registry the LAB raised inside the scenario — scenery pretending to + * be upstream, which is the thing this change removed. A registry the mesh runs and a registry the + * lab runs are different claims, and only the first exists in production. + */ +const ARTIFACT_STORE = + "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; +const registry = "127.0.0.1:5000"; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -66,28 +82,33 @@ async function mesh(command: string): Promise { return must(`docker exec mesh-control /mesh-control ${command}`); } -/** The bundle, pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const pinned of images) { - const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), pinned); - } - return text; +/** The bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } before(async () => { if (skip) return; const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {}); instanceId = raised.instanceId; - const first = raised.images[0]; - assert.ok(first, "the scenario stocked no images, so there is no registry to publish to"); - registry = first.slice(0, first.indexOf("/")); await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`); + // The mesh's artifact store, standing where the `registry` module would. Read back rather than + // assumed: a builder publishing into a registry that never came up fails several minutes later, + // as a manifest naming a blob nobody has. + await must( + `docker run -d --name mesh-registry --restart unless-stopped ` + + `-p ${registry}:5000 ${ARTIFACT_STORE}`, + ); + let serving = false; + for (let i = 0; i < 30 && !serving; i++) { + ({ ok: serving } = await on(`curl -sf http://${registry}/v2/ >/dev/null`)); + if (!serving) await new Promise((r) => setTimeout(r, 2_000)); + } + assert.ok(serving, "the mesh's artifact store never answered, so a build has nowhere to publish"); + // A module repository on the machine. Local rather than fetched, because what is under test is // the mesh's chain and not whether the lab can reach a forge. await must(`mkdir -p /root/shell/files`); diff --git a/test/integration/certificates.test.ts b/test/integration/certificates.test.ts index 81f4d46..37aa663 100644 --- a/test/integration/certificates.test.ts +++ b/test/integration/certificates.test.ts @@ -32,6 +32,14 @@ const SCENARIO = "a-public-name"; const MACHINE = "anchor"; const NAME = "photos.example"; const ACME = "/var/lib/acme"; +/** + * The ACME server under test, pulled by the machine over its uplink. + * + * It used to be served from a registry the lab raised inside the scenario. Nothing outside the lab + * has one, so an image only reachable there was a fiction — and this test is about a certificate + * being obtained over a real path. + */ +const AUTHORITY = "ghcr.io/letsencrypt/pebble:2.5.0"; let instanceId = ""; @@ -59,8 +67,7 @@ before(async () => { const instance = await raise(scenario, {}); instanceId = instance.instanceId; - const pebble = instance.images.find((r) => r.includes("pebble")); - assert.ok(pebble, `the scenario stocked no ACME server: ${instance.images.join(", ")}`); + const pebble = AUTHORITY; await must(`mkdir -p ${ACME}/cache`); diff --git a/test/integration/events.test.ts b/test/integration/events.test.ts index 1264d47..3ed2758 100644 --- a/test/integration/events.test.ts +++ b/test/integration/events.test.ts @@ -33,7 +33,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; @@ -97,17 +98,8 @@ async function must(command: string, timeoutMs?: number): Promise { * is not this one; matching by repository and rewriting to the digest this registry assigned is * what makes it applicable (the same rewrite mesh.test.ts does). */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const pinned of images) { - const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll( - new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), - pinned, - ); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } /** Read the trail back as parsed JSON lines. */ @@ -129,7 +121,8 @@ before(async () => { instanceId = raised.instanceId; // The node raises its substrate — store, broker and the rest — from the bundle, applied from a - // file because the digests are this registry's and are not known until it is up. + // file because the control plane's image is named by the ID this machine holds it under, + // which is not knowable until it has been handed over. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/harness.ts b/test/integration/harness.ts index 1e06547..8d82bc0 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -9,11 +9,126 @@ */ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts"; import { destroy, list } from "../../src/lifecycle/operate.ts"; import { diagramFromLive } from "../../src/diagram/from-live.ts"; import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts"; import type { Scenario } from "../../src/declaration/types.ts"; +import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImage } from "../../src/pinning.ts"; + +// --- the substrate bundle, and what its three images are on a real machine --------------------- + +/** + * The example bundle in mesh-host names a registry that no longer exists. + * + * `examples/substrate-first-node.lock` was written **for a target**, and the target was the lab: it + * pins `192.0.2.250:5000/…` because that is where the registry the lab used to raise served from. + * That registry is gone, so those three references name nothing. + * + * Two of them are ordinary third-party images and belong to the internet. Rather than invent + * digests here, they are the ones the mesh's own modules already pin — mesh-catalog's `postgres` + * and `lavinmq` — so the substrate's store and broker are literally the images the mesh runs. The + * third, mesh-control, exists in no registry at all and becomes the ID the machine holds it under. + * + * **The bundle itself should be fixed in mesh-host**, and this substitution deleted with it. It is + * here because the file lives in another repository and because a fixture that lies about where an + * image comes from is exactly what this change is removing. + */ +const UPSTREAM_STORE = + "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"; +const UPSTREAM_BROKER = + "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"; + +/** + * The substrate bundle as a machine should receive it. + * + * Third-party references become upstream ones, which the machine pulls over its uplink; ours + * become the ID the machine was handed. Nothing points inside the scenario any more, which is the + * whole of this change: what the bed proves about a bootstrap is now what would happen anywhere. + */ +export function substrateBundle(path: string, held: HeldImage[]): string { + let text = readFileSync(path, "utf8"); + text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE); + text = text.replaceAll( + /[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER); + return pinnedInto(text, held); +} + +/** + * The upstream reference for a third-party image, as the mesh's own catalogue pins it. + * + * A bed that writes a manifest by hand still has to name an image exactly — mesh-host refuses a + * tag, and rightly (novox/hq ADR 0006). While the lab had a registry the beds sidestepped that by + * naming a repository and letting the rewrite supply a digest; there is nothing to supply one now, + * so the digest has to be written down. + * + * These are the digests mesh-catalog's own modules pin, taken from `mesh-catalog/modules/*` — so a + * bed runs the image the mesh runs, and a bed that drifts from the catalogue is a bed testing a + * different postgres than the mesh ships. + */ +const UPSTREAM = new Map([ + ["alpine", "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"], + ["baserow/baserow", "baserow/baserow@sha256:834424a10413798567f76428f255dc259445b7f8dcec56598c05b4073bb2a124"], + ["cloudamqp/lavinmq", "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"], + ["eclipse-mosquitto", "eclipse-mosquitto@sha256:6f8d8a947c506f8a2290ec65cd4bd2bc7cb4d43fb5f6271f861cb013e2ef9797"], + ["ghcr.io/umami-software/umami", "ghcr.io/umami-software/umami@sha256:fa32d116cf20cad52cbc3fad9a63b46e7fa02299d8f967168eb453d49c476b4a"], + ["letta/letta", "letta/letta@sha256:1d2e0692514287c5ed1a483e14e16ed945f8632d315539f5e66373bb7d7c471b"], + ["lscr.io/linuxserver/radarr", "lscr.io/linuxserver/radarr@sha256:119aaa4a4f7349bcd2a136c5373a0d7925b5479915c7dfe0c0ad352db2a6d438"], + ["lscr.io/linuxserver/sonarr", "lscr.io/linuxserver/sonarr@sha256:c19aa4ecdf03d73e1d5c901da33744cb7eb4d921f89bafed1ca264601d7fa224"], + ["lscr.io/linuxserver/unifi-controller", "lscr.io/linuxserver/unifi-controller@sha256:fcd5d8b13a77a588c79c1b49e5fc9ad08115aa3bb1a3576c589c64908a68845f"], + ["minio/minio", "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2"], + ["mongo", "mongo@sha256:e3fa459b4f4b72f3257c67a23c145e250b8b5700f033860392c68539b998bbe3"], + ["ollama/ollama", "ollama/ollama@sha256:32931b46719f673c05fdbaa81ccb26da18ea4a1c57590a754874ab28ba269eb2"], + ["postgres", "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"], + ["redis", "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"], + ["registry", "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"], + ["synesthesiam/marytts", "synesthesiam/marytts@sha256:45970ecb3e21a2981c66c60563a70cf00be8e95c02565e7d74b3a73dcec7db2c"], +]); + +/** + * What a manifest's image reference becomes on the machine. + * + * Four cases, and the second one is the whole change: + * + * - **Ours** becomes the ID the machine holds it under. Nothing serves it, and nothing needs to. + * - **Anything already pinned by digest** is returned exactly as written. The machine pulls it + * from the internet, over its uplink, which is what a real machine does and what the lab spent + * a long time serving from a registry of its own instead. + * - **A bare repository a bed names by hand** is given the digest mesh-catalog pins for it, so a + * bed runs the image the mesh ships. A tag would be refused by mesh-host anyway. + * - **A tag this harness has never heard of** is passed through untouched, and said out loud. + * + * That last case is not politeness, it is a finding the lab's registry was hiding. Seven catalogue + * modules name `registry-api.…/novox/…:latest` — a TAG, which ADR 0006 forbids and mesh-host + * refuses. It never showed, because the rewrite replaced every reference with a digest the lab's + * registry had assigned, tag or not. There is nothing to replace it with now, and the honest + * outcome is that those modules fail to apply, saying exactly why, on the node that carries them — + * rather than an assertion here taking the whole bed down before it starts. + */ +export function onTheMachine(reference: string, held: HeldImage[]): string { + if (mustBeHandedOver(reference)) { + const found = referenceFor(held, repositoryOf(reference)); + assert.ok( + found, + `nothing loaded ${reference} onto the machines. They hold:\n ` + + held.map((i) => `${i.repository} ${i.reference}`).join("\n "), + ); + return found; + } + if (reference.includes("@sha256:")) return reference; + + const upstream = UPSTREAM.get(repositoryOf(reference)); + if (upstream) return upstream; + + console.log( + `UNPINNED: ${reference} names a tag, not a digest. The host will refuse it (novox/hq ` + + `ADR 0006). The lab's own registry used to paper over this by assigning a digest to ` + + `whatever was pushed; nothing does now. Fix the manifest, or add its digest to the ` + + `harness's UPSTREAM table.`, + ); + return reference; +} export interface Capability { usable: boolean; diff --git a/test/integration/lavinmq-bed.test.ts b/test/integration/lavinmq-bed.test.ts index 44a2ab8..e2326d5 100644 --- a/test/integration/lavinmq-bed.test.ts +++ b/test/integration/lavinmq-bed.test.ts @@ -41,7 +41,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -62,7 +63,7 @@ const NODE = "laptop"; const CONSUMER_LOGIN = "mesh_laptop_ping"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -88,22 +89,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -161,7 +155,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/local-model-bed.test.ts b/test/integration/local-model-bed.test.ts index 4dfc3e7..981734c 100644 --- a/test/integration/local-model-bed.test.ts +++ b/test/integration/local-model-bed.test.ts @@ -26,7 +26,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -44,7 +45,7 @@ const SCENARIO = "local-model-bed"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -73,20 +74,13 @@ async function meshTry(command: string): Promise<{ out: string; ok: boolean }> { return on(`docker exec mesh-control /mesh-control ${command}`); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -136,7 +130,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/mesh-grant-end-to-end.test.ts b/test/integration/mesh-grant-end-to-end.test.ts index 1543ef5..0d96850 100644 --- a/test/integration/mesh-grant-end-to-end.test.ts +++ b/test/integration/mesh-grant-end-to-end.test.ts @@ -23,7 +23,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -41,7 +42,7 @@ const SCENARIO = "redis-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -66,20 +67,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -122,7 +116,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 5f2923f..5c249db 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -21,10 +21,10 @@ import assert from "node:assert/strict"; import { existsSync, readFileSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; -import { pinnedInto, stillUnpinned } from "../../src/pinning.ts"; +import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts"; @@ -49,23 +49,27 @@ const skip = !capability.usable const SCENARIO = "two-nodes"; let instanceId = ""; -/** The scenario's own registry, which serves the images a module may mirror. */ -let registry = ""; -/** What that registry actually serves, by repository. */ -let stocked: string[] = []; - /** - * The pinned reference for one of the scenario's images. + * The MESH's own artifact store, once the registry module is running on the anchor. * - * By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and - * asking for it fails with "not found", which reads like a missing image rather than a naming - * convention. A digest is also what a declaration pins, so this is the reference a module would - * really carry. + * Not a registry the lab raised — there is no longer any such thing. A build publishes into the + * store the mesh itself runs, which is the only registry that exists outside this repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +const registry = "127.0.0.1:5000"; +/** + * The registry module's image, pinned upstream, pulled by the machine over its uplink. + * + * The digest mesh-catalog's `registry` module pins, so the store the mesh runs here is the store + * the mesh runs anywhere. + */ +const ARTIFACT_STORE = + "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; +/** The mesh's own images, as the machines hold them. */ +let held: HeldImage[] = []; + +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } function quote(s: string): string { @@ -179,23 +183,14 @@ async function settled(node: string, withinMs = 480_000): Promise { } /** - * The bundle, with every image reference pointed at this scenario's registry. + * The bundle, as a machine should receive it. * - * Matched by repository rather than by the whole reference, because the address and the digest - * both differ from whatever the committed bundle names — and a bundle that names the wrong - * registry is not wrong, it is built for a different target. + * The committed example was written for a target that had a registry the lab raised. Its two + * third-party images become upstream references the machine pulls itself; mesh-control, which + * exists in no registry, becomes the ID this machine was handed. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const pinned of images) { - const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll( - new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), - pinned, - ); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } /** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */ @@ -219,7 +214,7 @@ before(async () => { if (said.use === "restore") { instanceId = said.instanceId; const seconds = await returnTo(instanceId); - stocked = warmStock(instanceId).images; + held = warmStock(instanceId).images; // **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything // this suite started by hand is gone — the host most of all. Without it the mesh looks @@ -255,13 +250,11 @@ before(async () => { instanceId = raised.instanceId; // The first node raises everything from a file rather than from a bundle built into the binary, - // because the digests are this registry's and are not known until it is up. + // because the control plane's image is named by the ID this machine holds it under, which is not + // knowable until it has been handed over. + held = raised.images; await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`); - stocked = raised.images; - const first = raised.images[0]; - assert.ok(first, "the scenario stocked no images, so nothing can be mirrored"); - registry = first.slice(0, first.indexOf("/")); // A build machine, so anything here can ask the mesh to build something. Placed rather than // assumed: nothing else in this scenario would start one. @@ -279,7 +272,7 @@ before(async () => { if (warming) { // Snapshotted only now, with everything up: a state worth returning to is the one after the // part nobody wants to repeat. - await rememberStock(instanceId, stocked); + await rememberStock(instanceId, held); const warm = await keep(SCENARIO, instanceId); console.log(`warm: ${warm.instanceId} kept, against ` + Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", ")); @@ -608,13 +601,13 @@ test("a machine that fell behind catches up without being named", { skip, timeou }); test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async () => { - // Artifacts go to a registry, and the only registries that existed were raised by the lab or by - // the bootstrap bundle. A mesh had no way to run its own. + // Artifacts go to a registry, and a mesh had no way to run its own — the only one that existed + // was raised by the lab, which is to say it existed nowhere but here. // // **Named, not mirrored** (novox/hq 04-ISSUES/029). Mirroring publishes to the artifact store, // and the builder will not start without one — so a module that provides the store and builds // its own image asks the mesh to put an artifact into the thing that artifact is needed to - // create. It worked here only because the scenario's registry was already standing to receive + // create. It used to pass here only because the LAB's registry was already standing to receive // the push, which is exactly why a real first mesh would have found this and the lab did not. // // So the image is named by digest, the way the bundle names the three a first node starts from. @@ -626,7 +619,7 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async ( `"serves":{"artifact-store":{"port":5000}},` + `"resources":[` + `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` + - `{"id":"store","type":"container","name":"mesh-registry","image":"${pinned("registry")}",` + + `{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` + `"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` + `> /root/registry/module.json`); // **Added, not built** — and this is the half that proves the fix. Building needs a builder, @@ -677,7 +670,7 @@ test("a machine serves its internal name with a certificate the mesh issued", { // The name it was issued for is the one the mesh gave this machine. const named = await must("anchor", `openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` + - `docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` + + `docker run --rm -v /etc/mesh:/m ${ARTIFACT_STORE} sh -c ` + `"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`); assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`); @@ -1086,7 +1079,7 @@ test("a route is a grant: a workload is reached by the name it asked for", { `"listens":[{"port":8088,"from":"mesh","why":"the proxy reaches it here"}],` + `"resources":[{"id":"dir","type":"directory","path":"/etc/storefront","mode":"0755"},` + `{"id":"app","type":"container","name":"storefront",` + - `"image":"${pinned("registry")}","ports":["8088:5000"]}]}' > /tmp/storefront.json`); + `"image":"${ARTIFACT_STORE}","ports":["8088:5000"]}]}' > /tmp/storefront.json`); for (const f of ["frontdoor", "storefront"]) { await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`); await mesh(`module add /${f}.json`); @@ -1456,7 +1449,7 @@ test("a container reaches another machine by the name the mesh gave it", { await must("anchor", `printf %s '{"module":"resolves","version":"1",` + `"capabilities":["container-runtime"],` + `"resources":[{"id":"idle","type":"container","name":"resolves",` + - `"image":"${pinned("registry")}"}]}' > /tmp/resolves.json`); + `"image":"${ARTIFACT_STORE}"}]}' > /tmp/resolves.json`); await must("anchor", `docker cp /tmp/resolves.json mesh-control:/resolves.json`); await mesh("module add /resolves.json"); await mesh("assign laptop resolves"); @@ -1810,7 +1803,7 @@ test("the real modules resolve together, and compose a declaration a host accept // until it is built — so the file legitimately carries a placeholder, and composing a // declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is // what this test used to do. - const pinned = pinnedInto(raw, stocked); + const pinned = pinnedInto(raw, held); // What this scenario does not serve cannot be redirected, and a module still naming a // placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped // and said, rather than silently dropped: a planning test quietly covering four modules @@ -1934,8 +1927,8 @@ test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 for (const name of ["postgres", "gitea"]) { const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8"); // An image the mesh builds has no digest until it is built, and one it does not build belongs - // to whichever registry served it. Both are answered by this scenario's own registry. - const pinned = pinnedInto(raw, stocked); + // to whichever registry served it. Only the first is rewritten; the second is pulled. + const pinned = pinnedInto(raw, held); assert.deepEqual(stillUnpinned(pinned), [], `${name} still names an image nothing serves, so it could not start`); await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`); @@ -2021,7 +2014,7 @@ test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600 // keyspace, so the grant is a pattern — and the test is that the pattern means what the // manifest said, in both directions. const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8"); - const pinned = pinnedInto(raw, stocked); + const pinned = pinnedInto(raw, held); assert.deepEqual(stillUnpinned(pinned), [], "redis still names an image nothing serves, so it could not start"); await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`); diff --git a/test/integration/minio-grant-end-to-end.test.ts b/test/integration/minio-grant-end-to-end.test.ts index fbc860f..1fae1de 100644 --- a/test/integration/minio-grant-end-to-end.test.ts +++ b/test/integration/minio-grant-end-to-end.test.ts @@ -21,7 +21,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -42,7 +43,7 @@ const SCENARIO = "minio-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -67,20 +68,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -129,7 +123,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/objectstore.test.ts b/test/integration/objectstore.test.ts index 72f3adc..337025a 100644 --- a/test/integration/objectstore.test.ts +++ b/test/integration/objectstore.test.ts @@ -45,8 +45,16 @@ const ROOT_PASSWORD_FILE = "/var/lib/objectstore/root.secret"; const ENDPOINT = "http://127.0.0.1:9000"; let instanceId = ""; -/** The store's image, by digest, from the registry the scenario raised. */ -let storeImage = ""; +/** + * The store and the vendor's client, pinned upstream and pulled by the machine over its uplink. + * + * The store is the digest the mesh's own minio module pins, so this is the store the mesh runs. + * Both used to come from a registry the lab raised inside the scenario; no production mesh has + * one, so a test that could only fetch from it was proving something about the lab. + */ +const storeImage = + "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2"; +const clientImage = "minio/mc:RELEASE.2025-08-13T08-35-41Z"; function shellQuote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -141,18 +149,10 @@ before(async () => { const instance = await raise(scenario, {}); instanceId = instance.instanceId; - // From the registry the scenario raised, by digest. There is no route to a public registry from - // a documentation range, which is the point of the lab having its own. - const store = instance.images.find((r) => r.includes("minio/minio")); - const client = instance.images.find((r) => r.includes("minio/mc")); - assert.ok(store, `the scenario stocked no store image: ${instance.images.join(", ")}`); - assert.ok(client, `the scenario stocked no client image: ${instance.images.join(", ")}`); - storeImage = store; - // The client, taken out of the vendor's own image onto the machine. The provisioner drives it, - // so it has to be here — and taking it from the stocked image is what keeps this test off any - // public network. - await must(`docker create --name mc-source ${client}`); + // so it has to be here. The machine pulls the image itself, over its uplink, the way it pulls + // everything third-party. + await must(`docker create --name mc-source ${clientImage}`); await must(`docker cp mc-source:/usr/bin/mc /usr/local/bin/mc && chmod 755 /usr/local/bin/mc`); await must(`docker rm mc-source`); diff --git a/test/integration/openai-bed.test.ts b/test/integration/openai-bed.test.ts index f8b8494..f15d57d 100644 --- a/test/integration/openai-bed.test.ts +++ b/test/integration/openai-bed.test.ts @@ -24,7 +24,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -46,7 +47,7 @@ const MACHINE = "anchor"; const API_KEY = "sk-lab-openai-static-key-value-for-the-bed-only"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -75,20 +76,13 @@ async function meshTry(command: string): Promise<{ out: string; ok: boolean }> { return on(`docker exec mesh-control /mesh-control ${command}`); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -139,7 +133,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/postgres-grant-end-to-end.test.ts b/test/integration/postgres-grant-end-to-end.test.ts index 240bfdb..3ee0521 100644 --- a/test/integration/postgres-grant-end-to-end.test.ts +++ b/test/integration/postgres-grant-end-to-end.test.ts @@ -21,7 +21,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -39,7 +40,7 @@ const SCENARIO = "postgres-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -64,20 +65,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -120,7 +114,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/provider-on-backend-network.test.ts b/test/integration/provider-on-backend-network.test.ts index 11fb8a8..4b3f1b8 100644 --- a/test/integration/provider-on-backend-network.test.ts +++ b/test/integration/provider-on-backend-network.test.ts @@ -20,7 +20,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -38,7 +39,7 @@ const SCENARIO = "redis-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -63,20 +64,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -119,7 +113,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/provider-uses-mesh-credential.test.ts b/test/integration/provider-uses-mesh-credential.test.ts index 8724872..1fca4ba 100644 --- a/test/integration/provider-uses-mesh-credential.test.ts +++ b/test/integration/provider-uses-mesh-credential.test.ts @@ -24,7 +24,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -42,7 +43,7 @@ const SCENARIO = "redis-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -67,20 +68,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -123,7 +117,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/provisioner.test.ts b/test/integration/provisioner.test.ts index 715fd42..853d14b 100644 --- a/test/integration/provisioner.test.ts +++ b/test/integration/provisioner.test.ts @@ -34,8 +34,15 @@ const GRANTS = "/var/lib/postgres/grants"; const SUPER = "postgres://postgres:super@127.0.0.1:5432/postgres?sslmode=disable"; let instanceId = ""; -/** The postgres image, by digest, from the registry the scenario raised. */ -let image = ""; +/** + * The database, pinned upstream and pulled by the machine over its uplink. + * + * The same digest the mesh's own postgres module pins, so this is the database the mesh runs + * rather than a lookalike. It used to come from a registry the lab raised inside the scenario; + * nothing outside the lab has one, so what that proved about fetching an image was true only here. + */ +const image = + "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"; function shellQuote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -142,12 +149,6 @@ before(async () => { const instance = await raise(scenario, {}); instanceId = instance.instanceId; - // From the registry the scenario raised, by digest. There is no route to a public registry from - // a documentation range, which is the point of the lab having its own. - const stocked = instance.images.find((r) => r.includes("postgres")); - assert.ok(stocked, `the scenario stocked no postgres image: ${instance.images.join(", ")}`); - image = stocked; - await must( `docker run -d --name mesh-db -e POSTGRES_PASSWORD=super ` + `-p 127.0.0.1:5432:5432 ${image}`, diff --git a/test/integration/route-forwarding.test.ts b/test/integration/route-forwarding.test.ts index 33ed2ed..bb8fe23 100644 --- a/test/integration/route-forwarding.test.ts +++ b/test/integration/route-forwarding.test.ts @@ -37,7 +37,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -57,7 +58,7 @@ const NAME = "hello.example"; const PAGE = "hello from hello-web, routed by the mesh"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -83,22 +84,15 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -/** The pinned reference for one of the scenario's images, by repository. */ -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** The reference a manifest should carry, once this scenario has been raised. */ +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -/** The substrate bundle, its image references pointed at this scenario's own registry. */ -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */ +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -147,7 +141,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; // Raise the substrate — store, broker, control — from the bundle. await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); diff --git a/test/integration/runtime-restart-on-config.test.ts b/test/integration/runtime-restart-on-config.test.ts index fc6a155..6eaf500 100644 --- a/test/integration/runtime-restart-on-config.test.ts +++ b/test/integration/runtime-restart-on-config.test.ts @@ -24,7 +24,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -42,7 +43,7 @@ const SCENARIO = "grafana-node"; const MACHINE = "anchor"; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -67,20 +68,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`); - return found; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function tokenFrom(said: string): string { @@ -133,7 +127,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000); diff --git a/test/integration/underlay.test.ts b/test/integration/underlay.test.ts index fbe5b03..4dc3822 100644 --- a/test/integration/underlay.test.ts +++ b/test/integration/underlay.test.ts @@ -45,7 +45,7 @@ test("ADR 0016 — the lab provides the underlay and NOTHING of the overlay", { test("ADR 0016 — the declared address IS what the machine holds", { skip }, async () => { const { stdout } = await exec(instanceId, "home-server", ["ip", "-o", "-4", "addr", "show"]); - assert.match(stdout, /192\.168\.1\.135\/24/); + assert.match(stdout, /10\.99\.1\.135\/24/); }); test("design — raise waits for USABLE, not for the call to return", { skip, timeout: 120_000 }, async () => { @@ -75,7 +75,7 @@ test("ADR 0016 — a router is scenery: containers, while machines are virtual m test("design — NAT: a private address is not reachable from outside", { skip, timeout: 120_000 }, async () => { const { stdout } = await exec(instanceId, "anchor", [ - "sh", "-c", "ping -c1 -W2 192.168.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable", + "sh", "-c", "ping -c1 -W2 10.99.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable", ]); assert.equal(stdout.trim(), "unreachable"); }); @@ -139,7 +139,7 @@ test("the live diagram reads the hypervisor, and a VM's addresses are not lost", assert.ok(server, "home-server missing from the live picture"); assert.equal(server.kind, "machine"); assert.ok( - server.attachments.some((a) => a.addresses.some((address) => address.startsWith("192.168.1.135"))), + server.attachments.some((a) => a.addresses.some((address) => address.startsWith("10.99.1.135"))), `a virtual machine's addresses were not read back: ${JSON.stringify(server.attachments)}`, ); }); diff --git a/test/integration/whole-mesh-ace.test.ts b/test/integration/whole-mesh-ace.test.ts index 77393d1..39606aa 100644 --- a/test/integration/whole-mesh-ace.test.ts +++ b/test/integration/whole-mesh-ace.test.ts @@ -17,7 +17,7 @@ * apps unifi portainer * * Each committed module.json is LOADED from mesh-catalog (not hand-written); its container image - * references are rewritten to what this scenario's own registry serves by digest, and the co-located + * references of OURS are rewritten to the IDs the machine holds, and the co-located * host-port collisions are remapped at load time (see REMAP). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock @@ -31,7 +31,8 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); @@ -148,7 +149,7 @@ const REMAP: Record> = { }; let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -173,27 +174,13 @@ async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function repositoryFor(reference: string): string { - const withoutDigest = reference.split("@")[0] ?? reference; - const lastColon = withoutDigest.lastIndexOf(":"); - const lastSlash = withoutDigest.lastIndexOf("/"); - return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); - return found; -} - -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function loadManifest(name: string): { manifest: string; broker: boolean } { @@ -204,7 +191,7 @@ function loadManifest(name: string): { manifest: string; broker: boolean } { const remap = REMAP[name] ?? {}; for (const r of m.resources ?? []) { if (r.type !== "container") continue; - if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (typeof r.image === "string") r.image = pinned(r.image); if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); } const manifest = JSON.stringify(m); @@ -259,7 +246,7 @@ before(async () => { onProgress: (m) => console.log(`raise: ${m}`), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index 447c363..1d12af0 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -1,54 +1,76 @@ /** - * The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the - * whole-mesh rehearsal (novox/hq). Combines whole-mesh-novox.test.ts and whole-mesh-ace.test.ts. + * The FULL mesh in its REAL production shape: two segments, one access point, one overlay — and the + * first multi-segment whole-mesh bed. It rewrites the flat three-node whole-mesh-full (separate + * anchor, everything on one public segment) into what production actually is: * - * anchor — substrate ONLY (store, broker, control). - * novox — the 18-module novox set (whole-mesh-novox): providers, web apps, route-proxy, mailu, - * firewall, fail2ban. fail2ban is now HOSTABLE: the dry-run fixes (mesh-control/catalog - * main) changed its declared capability from the never-detected "intrusion-prevention" to - * "firewall", the detector every node with nft already advertises. - * ace — the 24-module ace set (whole-mesh-ace): the media/home stack; its /services/media - * library is pre-created so the ADR-0051 `accesses` resolve. + * hosting (public) home (private, behind a NAT access point) + * novox 192.0.2.20 — the ANCHOR: ace 10.99.1.10 the home server, media/IoT set + * substrate (store/broker/ shanks 10.99.1.20 workstation (light: portainer only) + * control) + the whole novox g14 10.99.1.30 workstation (light: portainer only) + * set + overlay hub + ingress * - * An overlay is placed across all three so cross-node `at` resolves. Each service node is - * self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and - * the shared broker/store on anchor. The four modules both nodes run (postgres, redis, mssql, - * portainer) are ADDED once and assigned to each node; each gets its own per-node broker account. + * There is NO separate anchor: novox IS the anchor. The substrate runs on novox, and novox also + * enrols as a node and receives its own service set — the substrate host and a service node at once. * - * THE DRY-RUN FIXES THIS RUN PROVES (mesh-control + mesh-catalog main): - * - fail2ban is HOSTABLE (capability "firewall"): it is assigned, not refused. Before, it declared - * the never-detected "intrusion-prevention" capability, so no node could host it and its - * un-hostable assignment refused the whole node's push. Hostability is the gate. Its service - * reaching active is a host concern this offline lab cannot meet — the VM ships nftables (so the - * firewall detector is advertised) but not fail2ban, and the isolated segment has no route to the - * package mirror, so pacman cannot fetch it. That is a documented lab gap, reported not gated. - * - the 7 tool-runtime credential modules (ace: plex, bazarr, ombi, home-assistant, nzbget, - * qbittorrent; novox: umami) now read their app credential from an operator-provided own-secret. - * This bed delivers a FAKE value for each through the real operator path (`secret accept`) - * BEFORE the push, and gates on the sidecar getting PAST its old "no credential" crash (it reads - * the delivered value). A fake value will not authenticate against the real app — the sidecar may - * still fail at app-auth, which is expected and does NOT gate; only the crash being GONE gates. + * TWO ACTS, AND THE BED NOW DISTINGUISHES THEM (novox/hq ADR 0067). * - * It otherwise tolerates the SAME known gaps the per-server beds proved and escalated (the credential - * sidecars' app-auth failures, photos/mailu, and firewall's oneshot nftables.service); it gates green - * on each node's CORE converging whole and on no NON-GAP resource failing to apply — i.e. the two - * node-plans converge together on one substrate. + * GENESIS — novox is brought into existence by `mesh-bootstrap`, the installer, run on the + * machine exactly as a person would run it on a bare one: preflight, load the carried + * control-plane image, write the bundle, apply, verify, enrol itself, install the registry + * module, push the control-plane image into it, reinstall the control plane as an ordinary + * module pinned to the digest that push produced, retire the temporary one. Afterwards novox + * is a WORKING MESH OF ONE, and this bed asserts exactly that before going any further. + * + * JOINING — ace, shanks and g14 then join a mesh that already exists: host binary, token, + * `enrol`, run the agent. No bootstrap, no substrate, no registry. novox is NOT enrolled again. + * + * The bed used to do neither. It applied the substrate bundle itself and looped enrolment over all + * four machines as one continuous operation — which got the order right by accident and modelled + * the wrong shape, and is why ADR 0067's own acceptance check ("the bed bootstraps through the + * installer rather than around it") went unmet. Genesis GATES joining: if it stops, the bed says + * which of the installer's ten steps it stopped at and goes no further, because a second machine + * joining a mesh that is not ready is a different failure and must not be mistaken for this one. + * + * THE THING THIS BED EXISTS TO PROVE (the flat beds never could): does the WireGuard overlay tunnel + * FORM across the access point? A home node (ace/shanks/g14) dials novox's PUBLIC hub endpoint + * 192.0.2.20:51820/udp OUT through the household gateway's masquerade; the handshake has to complete + * through that NAT and the keepalive has to hold the hole open. Phase A drives exactly this and + * verifies it — WireGuard handshake state AND a ping over the overlay from a home node to novox — + * BEFORE any heavy module lands, so the cross-segment-overlay verdict survives whatever the module + * convergence then does. Phase B converges the full node sets and reports per node. + * + * SUBSTRATE-ON-NOVOX PORT COLLISIONS (a real consequence of collapsing the anchor onto novox that the + * separate-anchor beds never hit): the substrate store binds 127.0.0.1:5432 and novox's postgres + * provider publishes 5432; the substrate broker binds 5671 + 127.0.0.1:5672 and novox's lavinmq + * provider publishes 5672. The two provider host publishes are REMAPPED off the substrate's ports + * (REMAP below); consumers reach the providers over the mesh network on the container port, so the + * host side is free to move. Reported as a topology finding. + * + * PERSISTENT RAISE. With MESH_LAB_KEEP set the instance is raised under a fixed id + * (whole-mesh-full-live) and NOT torn down — it is left standing and browsable. Without it the bed + * behaves like every other: raise in before(), destroy in after(). * * MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock + * MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap MESH_LAB_CATALOG=.../mesh-catalog/modules */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; -import { destroy, exec } from "../../src/lifecycle/operate.ts"; -import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll } from "./harness.ts"; +import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts"; +import { + bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH, +} from "../../src/lifecycle/place.ts"; +import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts"; +import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); +const installer = bootstrapBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; @@ -58,9 +80,68 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)" - : false; + : !installer || !existsSync(installer) + ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap (mesh-host `make " + + "bootstrap IMAGE=mesh-control:development`). The anchor is raised BY the installer now, " + + "so a run without one would be testing the procedure this bed exists to stop testing" + : false; const SCENARIO = "whole-mesh-full"; +/** novox hosts the substrate and the control plane; it is where `mesh` commands run. */ +const CONTROL = "novox"; +/** Every node in the mesh. novox is on hosting; the rest are behind the home gateway. */ +const NODES = ["novox", "ace", "shanks", "g14"]; +/** + * The machines that JOIN. novox is not one of them, and that is the distinction this bed was + * restructured to make: novox is brought into existence by the installer, which enrols it as part + * of genesis. Enrolling it again here would be a second identity the mesh does not know. + */ +const HOME_NODES = ["ace", "shanks", "g14"]; + +/** A checkout of the mesh's catalogue, on the anchor, for the installer to read manifests from. */ +const CATALOGUE_ON_MACHINE = "/opt/mesh-catalog"; + +/** + * The manifests `mesh-bootstrap` reads out of that checkout — and only those. + * + * Named here rather than pushing the whole repository because the whole repository is a hundred + * megabytes of `node_modules` and the installer opens exactly two files: mesh-host's + * `internal/bootstrap` RegistryModule and ControlPlaneModule. If it ever opens a third, this list + * is where the bed finds out, by the installer saying which manifest it could not read. + */ +const CATALOGUE_MODULES = ["registry", "mesh-control"]; + +/** + * Where this mesh keeps its own images, as the anchor reaches it. + * + * **Loopback, and that is a finding rather than a shortcut.** The mesh's registry is plain HTTP on + * purpose — it is reached over the mesh's own network, which is already the encrypted and + * authenticated thing — and a container runtime refuses a plain-HTTP registry at any address + * EXCEPT a loopback one unless it has been told to allow it. So genesis can push to 127.0.0.1:5000 + * with no configuration, and the reference the control-plane module is then pinned to is one only + * the anchor can pull. On this bed that is enough, because only the anchor runs the control plane. + * A mesh where a second machine had to pull it would need the runtimes told about the registry + * first, and nothing in the design says who does that. + */ +const MESH_REGISTRY = "127.0.0.1:5000"; + +/** + * ADR 0066 — the domain each public-facing node composes its routed names under. + * + * **The bed had none, so the ADR was untested by construction.** A module now contributes a `label` + * to `route` and nothing else; the mesh joins it to the node's public domain and the join is the + * whole feature. On a node with no public domain a labelled contribution composes to nothing — no + * host, no route — so every routed module on this bed was silently unreachable and the bed still + * went green. Two nodes face outward here; the workstations do not and get none, which is also part + * of the design being exercised. + * + * `.incus` rather than the real domains: this repository's beds name nothing routable. + */ +const PUBLIC_DOMAIN: Record = { novox: "novox.incus", ace: "zurag.incus" }; + +/** Keep the instance standing and browsable rather than tearing it down. */ +const KEEP = !!process.env["MESH_LAB_KEEP"]; +const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined); const catalogDir = process.env["MESH_LAB_CATALOG"] ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") @@ -74,41 +155,69 @@ const MEDIA_DIRS = [ type Mod = { name: string; containers: string[]; node?: boolean; runOnce?: string[] }; -/** The novox node's 17-module set (fail2ban dropped). CORE gates; the rest are documented gaps. */ +/** + * The novox set (feat/novox-conversions @ 431310f). The slug fix means only-office/de-spiegel/ + * amqp-email-forwarder now resolve (their minted login was over the 20-char cap before), so they + * are INCLUDED. CORE gates; the rest are reported gaps (documented in the whole-mesh-novox bed): + * umami (provisioner url/admin unset), mailu (nox-schema gaps), only-office/de-spiegel (new plain + * apps, boot secondary), amqp-email-forwarder (hard-coded AMQP vhost authz), firewall/fail2ban + * (offline lab cannot fetch the package). + */ const NOVOX: Mod[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, { name: "minio", containers: ["minio", "mesh-minio"] }, { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, { name: "mssql", containers: ["mssql", "mesh-mssql"] }, + { name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] }, + // ADR 0066: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or + // route-proxy is unresolvable and takes every routed module down with it. step-ca is that + // provider, on the anchor, at mesh scope. + { name: "step-ca", containers: ["step-ca"] }, + { name: "route-proxy", containers: ["route-proxy"] }, { name: "keycloak", containers: ["keycloak", "mesh-keycloak"] }, { name: "gitea", containers: ["gitea", "mesh-gitea"] }, { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, { name: "umami", containers: ["umami", "mesh-umami"] }, - { name: "photos", containers: ["photos", "mesh-photos"] }, + { name: "photos", containers: ["photos-server", "photos-admin-client", "photos-client-eef", "photos-client-filip"] }, { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, + { name: "novox.be", containers: ["novox-be"] }, + { name: "only-office", containers: ["office-novox-be"] }, + { name: "de-spiegel", containers: ["de-spiegel-novox-be"] }, + { name: "amqp-email-forwarder", containers: ["amqp-email-forwarder"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, + // Already installed, assigned and running — genesis needed it to publish the control plane's + // image. Left in the plan on purpose: registering the same manifest and assigning it again is + // what an operator's `module add` + `assign` would do on a mesh that already has it, and a + // module the installer put there had better survive being asked for a second time. It also keeps + // mesh-registry in the convergence report, where a reader expects to see it. { name: "registry", containers: ["mesh-registry"] }, - { name: "route-proxy", containers: ["route-proxy"] }, { name: "mailu", containers: [ - "mailu-resolver", "mailu-redis", "mailu-admindb", "mailu-admin", "mailu-imap", - "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", + "mailu-resolver", "mailu-redis", "mailu-admin", "mailu-imap", "mailu-smtp", + "mailu-antispam", "mailu-antivirus", "mailu-webmail", "mailu-webdav", "mailu-fetchmail", + "mailu-front", "mesh-mailu", ], }, { name: "firewall", containers: [], node: true }, { name: "fail2ban", containers: [], node: true }, ]; const CORE_NOVOX = new Set([ - "postgres", "redis", "minio", "mongodb", "mssql", - "keycloak", "gitea", "nextcloud", "invoicing", - "portainer", "verdaccio", "registry", "route-proxy", + "postgres", "redis", "minio", "mongodb", "mssql", "lavinmq", + "route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be", + "portainer", "verdaccio", "registry", +]); +const GAPS_NOVOX = new Set([ + "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", + // step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in + // mesh-control, and this bed is not the place to discover that a fix has not landed yet. What is + // gated is the half that is decided and cheap — see the ADR 0066 section at the end. + "step-ca", ]); -const GAPS_NOVOX = new Set(["umami", "photos", "mailu", "firewall", "fail2ban"]); -/** The ace node's 24-module set. */ +/** The ace media/home set. */ const ACE: Mod[] = [ { name: "postgres", containers: ["postgres", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, @@ -142,16 +251,27 @@ const CORE_ACE = new Set([ ]); const GAPS_ACE = new Set(["plex", "bazarr", "nzbget", "qbittorrent", "ombi", "home-assistant", "letta"]); +/** The two workstations run one light module each, to prove a real module converges and joins the overlay. */ +const LIGHT: Mod[] = [{ name: "portainer", containers: ["portainer", "mesh-portainer"] }]; +const CORE_LIGHT = new Set(["portainer"]); +const GAPS_LIGHT = new Set(); + const PLAN: { node: string; mods: Mod[]; core: Set; gaps: Set }[] = [ { node: "novox", mods: NOVOX, core: CORE_NOVOX, gaps: GAPS_NOVOX }, { node: "ace", mods: ACE, core: CORE_ACE, gaps: GAPS_ACE }, + { node: "shanks", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT }, + { node: "g14", mods: LIGHT, core: CORE_LIGHT, gaps: GAPS_LIGHT }, ]; /** - * Host-port remaps (per module — host ports are per-VM, so novox's and ace's never clash). Union of - * both per-server beds' remaps. + * Host-port remaps (per module; host ports are per-VM so novox's and ace's never clash across nodes). + * The two SUBSTRATE collisions are the new ones: postgres 5432 and lavinmq 5672 are moved off the + * substrate store/broker's host ports, which only exist on novox because that is where the substrate + * runs. The rest break the novox web/app host-port collisions (route-proxy fronts 80/443). */ const REMAP: Record> = { + postgres: { "5432": "127.0.0.1:15432:5432" }, + lavinmq: { "5672": "127.0.0.1:15673:5672" }, nextcloud: { "80": "8090:80" }, umami: { "3000": "3090:3000" }, invoicing: { "80": "8091:80", "9000": "9091:9000" }, @@ -160,15 +280,7 @@ const REMAP: Record> = { nzbget: { "6789": "6790:6789" }, }; -/** - * The 7 tool-runtime credential modules (novox/hq dry-run fix). Each now reads its app credential - * from an operator-provided own-secret (`name`, an own-secret path in its module.json), mounted into - * the sidecar at MESH_*_FILE. This bed delivers a FAKE value for each via the real operator path - * (`secret accept --from `) BEFORE the push, and asserts the sidecar - * gets PAST `crash` — the exact message its client threw when nothing was mounted. A fake value does - * not authenticate against the real app, so the sidecar may still fail later at app-auth (expected, - * not gated); only the "no credential" crash being GONE proves the wiring and gates. - */ +/** Operator-provided app credentials, delivered as fake values through the real `secret accept` path. */ const CREDENTIALS: { node: string; module: string; name: string; crash: string }[] = [ { node: "ace", module: "plex", name: "token", crash: "no Plex token" }, { node: "ace", module: "bazarr", name: "api-key", crash: "no Bazarr API key" }, @@ -179,8 +291,19 @@ const CREDENTIALS: { node: string; module: string; name: string; crash: string } { node: "novox", module: "umami", name: "admin", crash: "admin password is not set" }, ]; +/** Operator secrets for the credential modules that own-secret their whole app (mailu, de-spiegel). */ +const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [ + { node: "novox", module: "mailu", name: "secret-key", value: "0123456789abcdef0123456789abcdef" }, + { node: "novox", module: "mailu", name: "admin", value: "MailuAdminFakePass123" }, + { node: "novox", module: "mailu", name: "api-token", value: "mailuapitokenfake0123456789abcd" }, + { node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" }, + { node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" }, + { node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" }, + { node: "novox", module: "amqp-email-forwarder", name: "smtp-pass", value: "eef-pass-fake" }, +]; + let instanceId = ""; -let stocked: string[] = []; +let held: HeldImage[] = []; function quote(s: string): string { return `'${s.replaceAll("'", `'\\''`)}'`; @@ -201,31 +324,18 @@ async function must(machine: string, command: string, timeoutMs?: number): Promi return out; } +/** The control plane, a container on novox (the anchor). */ async function mesh(command: string, timeoutMs?: number): Promise { - return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs); + return must(CONTROL, `docker exec mesh-control /mesh-control ${command}`, timeoutMs); } -function repositoryFor(reference: string): string { - const withoutDigest = reference.split("@")[0] ?? reference; - const lastColon = withoutDigest.lastIndexOf(":"); - const lastSlash = withoutDigest.lastIndexOf("/"); - return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest; +/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ +function pinned(reference: string): string { + return onTheMachine(reference, held); } -function pinned(repository: string): string { - const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository); - assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`); - return found; -} - -function bundleFor(images: string[]): string { - let text = readFileSync(bundle, "utf8"); - for (const ref of images) { - const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@")); - const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\."); - text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref); - } - return text; +function bundleFor(images: HeldImage[]): string { + return substrateBundle(bundle, images); } function loadManifest(name: string): { manifest: string; broker: boolean } { @@ -236,7 +346,7 @@ function loadManifest(name: string): { manifest: string; broker: boolean } { const remap = REMAP[name] ?? {}; for (const r of m.resources ?? []) { if (r.type !== "container") continue; - if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image)); + if (typeof r.image === "string") r.image = pinned(r.image); if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); } const manifest = JSON.stringify(m); @@ -259,7 +369,7 @@ interface NodeState { } async function nodeState(node: string): Promise { - const asked = await on("anchor", `docker exec mesh-control /mesh-control status --json`); + const asked = await on(CONTROL, `docker exec mesh-control /mesh-control status --json`); if (!asked.ok) return { reached: false, applied: false, current: false, waiting: false, raw: asked.out }; let state: { wrong: { node: string; outcome: string; refused?: string; failed?: { id: string; error: string }[] }[]; @@ -293,64 +403,424 @@ async function psMapOf(node: string): Promise> { return map; } +/** + * ADR 0066: the internal CA is initialised FROM AN OPERATOR'S ROOT — it does not mint its own. + * + * So the bed has to be an operator. The material is made on the anchor with openssl and handed to + * the mesh through the real `secret accept` path, exactly as a person would: the mesh cannot invent + * a PEM, and the random 32 bytes it makes for an own-secret nobody supplied would leave step-ca + * crash-looping on a root key that is not a key. + */ +async function deliverCaRoot(): Promise { + const made = await on(CONTROL, [ + "set -e", + "mkdir -p /tmp/ca && cd /tmp/ca", + // No trailing newline on a password file: step-ca reads the file as the password itself. + "openssl rand -hex 16 | tr -d '\\n' > key-password", + "openssl ecparam -genkey -name prime256v1 -out root.unenc", + "openssl ec -in root.unenc -aes256 -passout file:key-password -out root.key", + "rm -f root.unenc", + "openssl req -x509 -new -key root.key -passin file:key-password -sha256 -days 3650" + + ` -out root.crt -subj "/CN=Mesh Internal CA/O=Novox Mesh Lab"`, + // Readable by the control plane, which is not root. Its image is FROM scratch and runs as + // 65534, and `docker cp` keeps the ownership and mode a file had outside — openssl writes a + // private key 0600 root-owned, so the copy landed unreadable and `secret accept` failed with + // `open /ca-root-key: permission denied`. The CA then crash-looped on a root it never got. + // Chowning it inside the container is not available: there is no shell in there to do it with. + // + // Safe here and nowhere else: these three exist for the seconds between being written and + // being sealed to the machine, on a lab node, for a CA thrown away with the scenario. + "chmod 0644 /tmp/ca/root.crt /tmp/ca/root.key /tmp/ca/key-password", + "docker cp /tmp/ca/root.crt mesh-control:/ca-root-cert", + "docker cp /tmp/ca/root.key mesh-control:/ca-root-key", + "docker cp /tmp/ca/key-password mesh-control:/ca-root-key-password", + ].join("\n"), 180_000); + if (!made.ok) { + console.log(`CA ROOT NOT MADE on ${CONTROL}:\n${made.out.split("\n").slice(-8).join("\n")}`); + return false; + } + for (const [name, file] of [ + ["root-cert", "/ca-root-cert"], + ["root-key", "/ca-root-key"], + ["root-key-password", "/ca-root-key-password"], + ] as const) { + try { + await mesh(`secret accept ${CONTROL} step-ca ${name} --from ${file}`); + } catch (err) { + console.log(`CA ROOT ACCEPT FAILED (${name}): ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); + return false; + } + } + return true; +} + +/** What a module's manifest says its route label is, or "" if it contributes no route. */ +function routeLabelOf(name: string): string { + const path = resolve(catalogDir, name, "module.json"); + const m = JSON.parse(readFileSync(path, "utf8")) as { + contributes?: { route?: { label?: string } }; + }; + return m.contributes?.route?.label ?? ""; +} + +/** A node's overlay (mesh0) address, or "" if it has none yet. */ +async function overlayAddr(node: string): Promise { + const out = (await on(node, `ip -4 -o addr show mesh0 2>/dev/null | awk '{print $4}' | cut -d/ -f1`)).out; + return out.split("\n").map((l) => l.trim()).find(Boolean) ?? ""; +} + +// ================================================================================================== +// PHASE 1 — GENESIS. novox is brought into existence by the installer. +// ================================================================================================== + +/** What genesis did, or where it stopped. */ +interface GenesisResult { + ok: boolean; + /** `step 7 of 10, registry` — the installer's own words, so the bed reports the cause. */ + step: string; + why: string; + report: string[]; +} + +/** The step a failed `mesh-bootstrap` names, as it prints it, or "" if it named none. */ +function stepIn(said: string): string { + return said.match(/^mesh-bootstrap: (step \d+ of \d+, [a-z-]+):/m)?.[1] ?? ""; +} + +/** + * Put on the anchor what the installer needs to read, and run it. + * + * **This is the whole of what changed, and it is not a refactor.** The bed used to apply the + * substrate bundle itself, by hand, and then enrol four machines in one loop. It got the order + * right by accident and it modelled the wrong shape: an install procedure that exists only as a + * test fixture is exercised by whoever writes tests and never by whoever installs, which is why + * every bootstrap fault this year was found late (novox/hq ADR 0067). The anchor is now raised by + * running the same program a bare machine runs, and the bed only reads the result. + * + * It is run up to three times. Not to paper over a failure — every attempt's failing step is + * printed — but because `mesh-bootstrap` is idempotent by design and says so, and because the one + * thing here that fails for a reason which goes away by itself is a pull: the store, the broker and + * the registry come from the internet, through a household gateway's masquerade, and Docker Hub + * rate-limiting an anonymous pull is not this mesh's fault. A re-run is the retry, and it is the + * retry the installer's own documentation names. + */ +async function genesis(images: HeldImage[]): Promise { + const report: string[] = ["================ GENESIS: novox becomes a mesh of one ================"]; + const stop = (step: string, why: string): GenesisResult => { + report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`); + return { ok: false, step, why, report }; + }; + + // The installer, beside the host binary. Everything else on this machine was placed by `raise`; + // this one is placed here because only the anchor is bootstrapped. + const name = await instanceNameOf(instanceId, CONTROL); + const version = await placeBootstrap(name, CONTROL, installer as string, + (m) => console.log(`genesis:${m}`)); + report.push(` installer ${version} at ${BOOTSTRAP_PATH}`); + + // The catalogue. `mesh-bootstrap --catalog` reads manifests from a CHECKOUT on the machine, + // because at this moment the mesh has no forge, no build machine and — until step 7 finishes — + // no registry. A manifest is a file, and somebody has to have put it there. + await must(CONTROL, `mkdir -p ${CATALOGUE_MODULES.map((m) => `${CATALOGUE_ON_MACHINE}/modules/${m}`).join(" ")}`); + for (const module of CATALOGUE_MODULES) { + const from = resolve(catalogDir, module, "module.json"); + assert.ok(existsSync(from), `the catalogue has no ${module}/module.json at ${from}`); + await push(instanceId, CONTROL, from, `${CATALOGUE_ON_MACHINE}/modules/${module}/module.json`); + } + report.push(` catalogue ${CATALOGUE_MODULES.join(", ")} at ${CATALOGUE_ON_MACHINE}`); + + // The substrate TEMPLATE — not the bundle. The installer produces the bundle from it: it replaces + // the control plane's image with the id of the image it carries, renames that container + // `temp-mesh-control`, and writes the result where a person can read it. + // + // Two substitutions still happen here, and both belong to the bed rather than to the installer. + // The example names three images at a registry the lab no longer raises: the store and the broker + // become the upstream references mesh-catalog pins (harness), and the machine pulls them over its + // uplink like any first node. The third, mesh-control, is deliberately LEFT naming that dead + // registry — the installer overwrites it, and leaving it proves that it does. + // + // And the broker's advertised address. The template hardcodes 192.0.2.10:5671, the old + // separate-anchor address; a token carries MESH_BROKER_ADDRESS verbatim as the endpoint an + // enrolling node dials, so with the substrate on novox it must be novox's own public address or + // every node would enrol against a dead one. The installer refuses to guess this and says so + // loudly, which is right — it does not know what this machine is called from outside. + const template = bundleFor(images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671"); + const local = join(tmpdir(), `mesh-lab-substrate-${process.pid}.lock`); + writeFileSync(local, template); + await push(instanceId, CONTROL, local, "/tmp/substrate-template.lock"); + + const command = [ + BOOTSTRAP_PATH, + `--bundle /tmp/substrate-template.lock`, + `--catalog ${CATALOGUE_ON_MACHINE}`, + `--node ${CONTROL}`, + `--registry ${MESH_REGISTRY}`, + `--host ${HOST_PATH}`, + // The lab has no unit to supervise the host with, and the installer refuses to invent one — a + // unit file is a packaging decision. This is the arrangement it offers instead, and it is loud + // about what it is: a host started this way does not survive a reboot. + `--host-in-background`, + ].join(" "); + + let said = ""; + let step = ""; + for (let attempt = 1; attempt <= 3; attempt++) { + const ran = await on(CONTROL, command, 2_400_000); + said = ran.out; + console.log(`\n---- mesh-bootstrap on ${CONTROL} (attempt ${attempt}) ----\n${said}`); + if (ran.ok) { + step = ""; + break; + } + step = stepIn(said); + if (attempt < 3) { + console.log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; ` + + `re-running in 30s — every step it already did will say so`); + await new Promise((r) => setTimeout(r, 30_000)); + } + } + if (step) { + return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n")); + } + + // ------------------------------------------------------------------------------------------ + // And now the only thing that matters: is novox a WORKING MESH OF ONE? Asked of the machine, + // never inferred from the installer exiting zero (novox/hq ADR 0018). + // ------------------------------------------------------------------------------------------ + + // 1. The control plane answers. Asked of the PERMANENT container by name — `status` opens all + // three stores, so a reply proves the connections it was given are the substrate's own. + const answered = await on(CONTROL, `docker exec mesh-control /mesh-control status`, 60_000); + report.push(` control plane ${answered.ok ? answered.out.split("\n")[0] : "NO ANSWER"}`); + if (!answered.ok) return stop("after step 10", `mesh-control does not answer:\n${answered.out}`); + + // 2. The registry replies on /v2/ — the registry API's own "yes, I am one and I am ready". A + // container that is up is not a registry that serves. + const v2 = await on(CONTROL, + `curl -s -o /dev/null -w '%{http_code}' --max-time 10 http://${MESH_REGISTRY}/v2/`); + const v2Code = v2.out.trim(); + report.push(` registry /v2/ ${v2Code || "no answer"}`); + if (v2Code !== "200") return stop("after step 10", `the mesh's own registry answered ${v2Code || "nothing"}`); + + // 3. THE PIVOT COMPLETED. ADR 0067 states this check in as many words: after installing, the + // running control plane's image is pinned by a digest THE MESH'S OWN REGISTRY ASSIGNED — not + // by an image id. If it is still an image id, the substrate's container is what is running, + // nothing was published, and this mesh can never roll out its own upgrades. + const pinnedTo = (await on(CONTROL, `docker inspect --format '{{.Config.Image}}' mesh-control`)) + .out.trim(); + report.push(` pinned to ${pinnedTo || "(nothing)"}`); + if (/^sha256:[0-9a-f]{64}$/.test(pinnedTo)) { + return stop("after step 10", + `mesh-control is running from ${pinnedTo}, which is an IMAGE ID — the digest of the image's ` + + `own configuration, which no registry ever served. The pivot did not happen: what is ` + + `running is the image the installer carried, not one this mesh published, so this mesh ` + + `cannot upgrade itself (novox/hq ADR 0067, "the pivot completed").`); + } + if (!new RegExp(`^${MESH_REGISTRY.replaceAll(".", "\\.")}/mesh-control@sha256:[0-9a-f]{64}$`) + .test(pinnedTo)) { + return stop("after step 10", + `mesh-control is running from ${pinnedTo || "nothing this bed could read"}, which is not a ` + + `digest assigned by ${MESH_REGISTRY}.`); + } + + // 3b. And the registry really serves it, asked of the registry rather than of the container. A + // reference is a claim; a tag list is the registry agreeing. + const tags = await on(CONTROL, + `curl -s --max-time 10 http://${MESH_REGISTRY}/v2/mesh-control/tags/list`); + report.push(` registry holds ${tags.out.trim() || "nothing"}`); + if (!tags.out.includes("genesis")) { + return stop("after step 10", + `${MESH_REGISTRY} does not serve mesh-control, so the digest the container is pinned to ` + + `names an image nothing can pull: ${tags.out.trim()}`); + } + + // 4. The temporary control plane is GONE. Two control planes is the half-finished state, and the + // name is the audit: a machine running mesh-control and not temp-mesh-control has pivoted. + const temp = await on(CONTROL, `docker inspect --format '{{.State.Status}}' temp-mesh-control`); + report.push(` temp-mesh-control ${temp.ok ? `STILL HERE (${temp.out.trim()})` : "gone"}`); + if (temp.ok) { + return stop("after step 10", + `temp-mesh-control is still ${temp.out.trim()}. Two control planes are consuming this ` + + `mesh's broker queues; neither is wrong and the pivot is not finished.`); + } + + // 5. And the mesh has heard from its one node. Everything the join phase does next depends on it. + const nodes = await on(CONTROL, `docker exec mesh-control /mesh-control node list`); + report.push(` node list ${nodes.out.trim().split("\n").join(" | ")}`); + const line = nodes.out.split("\n").map((l) => l.trim()).find((l) => l.startsWith(`${CONTROL} `)); + if (!line || !/^\S+\s+here\b/.test(line)) { + return stop("after step 10", + `the mesh has not heard from ${CONTROL}: ${line ?? "it has no record of it at all"}`); + } + + report.push(`\nVERDICT: ${CONTROL} is a working mesh of one, bootstrapped through the installer.`); + return { ok: true, step: "", why: "", report }; +} + +// ================================================================================================== +// PHASE 2 — JOINING. Everything else is a machine joining a mesh that already exists. +// ================================================================================================== + +/** + * ace, shanks and g14 join. Host binary plus a token — no bootstrap, no substrate, no registry. + * + * **novox is not in this loop.** It was enrolled by the installer, as part of becoming a mesh, and + * enrolling it again would present the mesh with a second identity for a node it already knows — + * which `mesh-host enrol` refuses, and rightly. + * + * The home nodes reach novox's public 192.0.2.20:5671 by dialling OUT through the household + * gateway, so the enrol itself is the first proof that outbound home→public works. + */ +async function joinTheMesh(): Promise { + // ADR 0066: said as soon as the record exists, because everything routed is composed from it. A + // node that faces the outside has one; the workstations do not, and are given none. The anchor's + // is set here rather than in genesis because it is a fact about the mesh, not part of raising + // one — and the installer has an opinion about neither. + const anchorDomain = PUBLIC_DOMAIN[CONTROL]; + if (anchorDomain) await mesh(`node public-domain ${CONTROL} ${anchorDomain}`); + + for (const machine of HOME_NODES) { + await mesh(`node add ${machine}`); + const domain = PUBLIC_DOMAIN[machine]; + if (domain) await mesh(`node public-domain ${machine} ${domain}`); + const token = tokenFrom(await mesh(`token issue --node ${machine}`)); + const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000); + assert.match(said, new RegExp(`enrolled as ${machine}`), said); + await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + } +} + before(async () => { if (skip) return; assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), + ...(FIXED_ID ? { instanceId: FIXED_ID } : {}), }); instanceId = raised.instanceId; - stocked = raised.images; + held = raised.images; + console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`); - await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`); - await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000); - const up = await must("anchor", `docker ps --format '{{.Names}}'`); - for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) { - assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`); + // ---- PHASE 1, and it GATES phase 2 --------------------------------------------------------- + // + // Caught rather than allowed to propagate, so that a failure BEFORE the installer ran — a + // manifest that is not where the bed thought, a binary that would not copy — is reported in the + // same shape as one the installer itself named, instead of as a bare stack trace from a helper. + let genesisResult: GenesisResult; + try { + genesisResult = await genesis(raised.images); + } catch (err) { + genesisResult = { + ok: false, + step: "getting the anchor ready to be bootstrapped — the installer never ran", + why: (err as Error).message, + report: ["================ GENESIS: novox becomes a mesh of one ================"], + }; } + console.log(genesisResult.report.join("\n")); + assert.ok(genesisResult.ok, + `GENESIS FAILED — ${genesisResult.step || "no step named"}.\n\n${genesisResult.why}\n\n` + + `No other machine was asked to join. A second machine joining a mesh that is not ready is a ` + + `different failure with a different cause, and running it now would bury this one under it.\n\n` + + genesisResult.report.join("\n")); - for (const machine of ["anchor", "novox", "ace"]) { - await mesh(`node add ${machine}`); - const token = tokenFrom(await mesh(`token issue --node ${machine}`)); - const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`); - assert.match(said, new RegExp(`enrolled as ${machine}`), said); - await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); - } + // ---- PHASE 2 -------------------------------------------------------------------------------- + await joinTheMesh(); // The operator provides ace's media library (ADR 0051 accesses confirm the paths, create nothing). await must("ace", `mkdir -p ${MEDIA_DIRS.join(" ")}`); -}, { timeout: 3_000_000 }); +}, { timeout: 5_400_000 }); after(async () => { + if (KEEP) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 900_000 }); -test("both server sets converge together on one substrate", { skip, timeout: 3_600_000 }, async () => { - // Overlay across all three, so every node's private address exists and cross-node `at` resolves. - await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab"); - await mesh("overlay place novox --site lab"); - await mesh("overlay place ace --site lab"); - await mesh("assign anchor networking"); - await mesh("assign novox networking"); - await mesh("assign ace networking"); +test("the full mesh forms across the access point and both server sets converge", { + skip, timeout: 5_400_000, +}, async () => { + // ================================================================================================ + // PHASE A — THE HEADLINE. Place the overlay (hub on novox at its public endpoint; the home nodes + // dial out, no endpoint of their own), assign networking to every node, push, and VERIFY the tunnel + // forms ACROSS the gateway. This runs BEFORE any heavy module, so the cross-segment-overlay verdict + // is captured whatever the module convergence then does. + // ================================================================================================ + await mesh("overlay place novox --hub --endpoint 192.0.2.20:51820 --site hosting"); + for (const node of HOME_NODES) await mesh(`overlay place ${node} --site home`); + for (const node of NODES) await mesh(`assign ${node} networking`); + for (const node of NODES) { + try { + await mesh(`push ${node}`, 180_000); + } catch (err) { + console.log(`networking push rejected (${node}): ${(err as Error).message.split("\n").slice(0, 4).join(" | ")}`); + } + } - // Add every unique module ONCE (the four shared modules are added once, assigned to each node), then - // issue a per-node broker account and assign, resiliently. - const added = new Map(); // name -> needs broker + // Give the home nodes time to dial the hub and complete a handshake through the NAT. + const overlay: Record = {}; + const deadline = Date.now() + 300_000; + while (Date.now() < deadline) { + for (const node of NODES) if (!overlay[node]) overlay[node] = await overlayAddr(node); + if (NODES.every((n) => overlay[n])) break; + await new Promise((r) => setTimeout(r, 8000)); + } + // A little longer for handshakes to settle (keepalive interval). + await new Promise((r) => setTimeout(r, 30000)); + + const overlayReport: string[] = ["================ CROSS-SEGMENT OVERLAY (the headline) ================"]; + for (const node of NODES) overlayReport.push(` ${node.padEnd(8)} mesh0 = ${overlay[node] || "NONE"}`); + + // The hub's WireGuard peers and their handshakes, from novox. + const hubWg = (await on("novox", `wg show 2>&1 || echo 'wg tool absent'`)).out; + overlayReport.push(`\n---- novox (hub) wg show ----\n${hubWg}`); + + // From each home node: its wg peer state (endpoint should be 192.0.2.20:51820, with a recent + // handshake) AND a ping to novox's overlay address — the functional proof the tunnel carries + // traffic across the gateway. + const overlayFormed: Record = {}; + const novoxOverlay = overlay["novox"] ?? ""; + for (const node of HOME_NODES) { + const wg = (await on(node, `wg show 2>&1 || echo 'wg tool absent'`)).out; + const handshake = (await on(node, `wg show all latest-handshakes 2>/dev/null | awk '{print $2}' | sort -rn | head -1`)).out.trim(); + const ping = novoxOverlay + ? await on(node, `ping -c 3 -W 2 ${novoxOverlay} 2>&1 | tail -3`) + : { out: "novox has no overlay address to ping", ok: false }; + const handshakeSecs = Number(handshake) || 0; + // Formed = we can reach novox over the overlay from this home node (traffic across the NAT). + overlayFormed[node] = ping.ok; + overlayReport.push(`\n---- ${node} (home) ----`); + overlayReport.push(wg.split("\n").map((l) => ` ${l}`).join("\n")); + overlayReport.push(` latest-handshake epoch: ${handshake || "none"}${handshakeSecs ? "" : " (no handshake recorded)"}`); + overlayReport.push(` ping novox(${novoxOverlay}) over overlay: ${ping.ok ? "REPLIES" : "NO REPLY"}`); + overlayReport.push(ping.out.split("\n").map((l) => ` ${l}`).join("\n")); + } + const anyHomeFormed = HOME_NODES.some((n) => overlayFormed[n]); + const allHomeFormed = HOME_NODES.every((n) => overlayFormed[n]); + overlayReport.push(`\nVERDICT: overlay across the access point ${allHomeFormed ? "FORMED for all home nodes" : anyHomeFormed ? "FORMED for some home nodes" : "DID NOT FORM"}.`); + const overlaySummary = overlayReport.join("\n"); + console.log(overlaySummary); + + // ================================================================================================ + // PHASE B — converge the full node sets on top of the overlay. + // ================================================================================================ + const added = new Map(); async function ensureAdded(name: string): Promise { const known = added.get(name); if (known !== undefined) return known; const { manifest, broker } = loadManifest(name); - await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await must(CONTROL, `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`); await mesh(`module add /${name}.json`); added.set(name, broker); return broker; } - const assigned: Record> = { novox: new Set(), ace: new Set() }; - const refused: Record = { novox: [], ace: [] }; + const assigned: Record> = { novox: new Set(), ace: new Set(), shanks: new Set(), g14: new Set() }; + const refused: Record = { novox: [], ace: [], shanks: [], g14: [] }; for (const { node, mods } of PLAN) { for (const { name } of mods) { try { @@ -366,20 +836,14 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 } } - // Operator-provided app credentials (novox/hq dry-run fix). BEFORE the push, hand the mesh a FAKE - // value for each of the 7 credential modules through the real operator path — `secret accept`, - // which seals the value to the node and records it as `accepted` (the mesh will not invent one). - // The push then delivers it to the sidecar's own-secret path. The `--from` file is staged into the - // mesh-control container (one file per distinct secret name). A module the node could not host is - // skipped (its secret has nowhere to go). + // Operator-provided app credentials (own-secrets), delivered as fake values through `secret accept`. const credentialDelivered = new Map(); for (const name of new Set(CREDENTIALS.map((c) => c.name))) { - await must("anchor", `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); + await must(CONTROL, `printf %s ${quote(`fake-${name}-value`)} > /tmp/fake-${name} && docker cp /tmp/fake-${name} mesh-control:/fake-${name}`); } for (const c of CREDENTIALS) { if (!assigned[c.node]!.has(c.module)) { credentialDelivered.set(`${c.node}/${c.module}`, false); - console.log(`CREDENTIAL SKIPPED ${c.node}/${c.module}: not assigned, nowhere to deliver`); continue; } try { @@ -390,40 +854,51 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 console.log(`CREDENTIAL ACCEPT FAILED ${c.node}/${c.module}: ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`); } } - - // ONE push per node. - const pushError: Record = { novox: "", ace: "" }; - for (const node of ["novox", "ace"]) { + // Whole-app own-secrets (mailu/de-spiegel/amqp-email-forwarder). + for (const s of OPERATOR_SECRETS) { + if (!assigned[s.node]!.has(s.module)) continue; try { - await mesh(`push ${node}`, 240_000); + const inControl = `/secret-${s.module}-${s.name}`; + await must(CONTROL, `printf %s ${quote(s.value)} > /tmp${inControl} && docker cp /tmp${inControl} mesh-control:${inControl}`); + await mesh(`secret accept ${s.node} ${s.module} ${s.name} --from ${inControl}`); } catch (err) { - pushError[node] = (err as Error).message; - console.log(`PUSH REJECTED (${node}):\n${pushError[node]}`); + console.log(`OPERATOR SECRET FAILED ${s.node}/${s.module}/${s.name}: ${(err as Error).message.split("\n").slice(0, 2).join(" | ")}`); } } - // Wait for both nodes' CORE containers to come up (they pull concurrently from the one registry). - const psMaps: Record> = { novox: new Map(), ace: new Map() }; + // ADR 0066: the CA's root, before the push that would otherwise deliver a random 32 bytes for it. + const caRootDelivered = assigned["novox"]!.has("step-ca") ? await deliverCaRoot() : false; + if (!caRootDelivered) console.log("ADR 0066: no operator root delivered; step-ca cannot initialise."); + + // ONE push per node (workstations first — cheap — then the heavy service nodes). + const pushError: Record = {}; + for (const node of ["shanks", "g14", "novox", "ace"]) { + try { + await mesh(`push ${node}`, 300_000); + } catch (err) { + pushError[node] = (err as Error).message; + console.log(`PUSH REJECTED (${node}):\n${pushError[node]!.split("\n").slice(0, 6).join("\n")}`); + } + } + + // Wait for each node's CORE containers to come up (all nodes pull concurrently from the one registry). + const psMaps: Record> = { novox: new Map(), ace: new Map(), shanks: new Map(), g14: new Map() }; for (const { node, mods, core } of PLAN) { if (pushError[node]) continue; const coreContainers = mods.filter((m) => core.has(m.name) && assigned[node]!.has(m.name)).flatMap((m) => m.containers); - const until = Date.now() + 2_700_000; + const until = Date.now() + 3_000_000; while (Date.now() < until) { psMaps[node] = await psMapOf(node); if (coreContainers.every((c) => (psMaps[node]!.get(c) ?? "").startsWith("Up"))) break; await new Promise((r) => setTimeout(r, 10000)); } } - await new Promise((r) => setTimeout(r, 20000)); // let first-boot bounces settle + await new Promise((r) => setTimeout(r, 20000)); // ================================================================================================ - // Per-node report + gating. GREEN = each node's push accepted, every CORE module converged whole, - // no NON-GAP resource failed to apply, fail2ban is hostable (assigned, not refused), and every - // credential sidecar advanced past its "no credential" crash. Tolerated: the credential sidecars' - // app-auth failures (bogus fake value), photos/mailu, firewall's oneshot nftables.service, and - // fail2ban's package (the offline lab cannot fetch it — a documented host gap). + // Per-node convergence report. // ================================================================================================ - const users = (await on("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; + const users = (await on("novox", `docker exec mesh-broker lavinmqctl list_users 2>&1`)).out; const allProblems: string[] = []; const report: string[] = ["================ FULL MESH CONVERGENCE ================"]; @@ -435,7 +910,7 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 const failedResources = st.wrong?.failed ?? []; report.push(`\n---- node ${node}: reached=${st.reached} applied=${st.applied} current=${st.current} waiting=${st.waiting} ----`); - if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node].split("\n").slice(0, 6).join("\n ")}`); + if (pushError[node]) report.push(` PUSH REJECTED: ${pushError[node]!.split("\n").slice(0, 6).join("\n ")}`); if (st.wrong) { report.push(` NODE WRONG: outcome=${st.wrong.outcome}`); for (const f of failedResources) report.push(` failed ${f.id}: ${f.error}`); @@ -445,19 +920,19 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 const coreFailures: string[] = []; for (const mod of mods) { if (!assigned[node]!.has(mod.name)) continue; + if (mod.node) { + report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${core.has(mod.name) ? "CORE" : "gap "} node-service`); + continue; + } const states = mod.containers.map((c) => `${c}:${running(c) ? "UP" : (psMap.get(c) ?? "MISSING")}`); const ok = mod.containers.every(running) && (mod.runOnce ?? []).every(ranOnce); const tag = core.has(mod.name) ? (ok ? "OK " : "FAIL") : (ok ? "ok " : "GAP "); - report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(15)} ${tag} ${states.join(" ")}`); + report.push(` ${core.has(mod.name) ? "*" : " "} ${mod.name.padEnd(20)} ${tag} ${states.join(" ")}`); if (core.has(mod.name) && !ok) coreFailures.push(mod.name); } const issuedHere = mods.filter((m) => new RegExp(`${node}-${m.name}\\b`).test(users)).length; report.push(` broker accounts: ${issuedHere} present for ${node}`); - // Gate: push accepted, all CORE up, no NON-GAP resource failed. A failed resource names its - // owning module inside the error (`applying "firewall.load": …`), not in `id` (which is the outer - // "apply" key), so the owner is extracted from either — and a failure owned by a KNOWN_GAP module - // (firewall's oneshot nftables.service) is tolerated. const gapOwnerOf = (f: { id: string; error: string }): string => { const m = f.error.match(/applying "([^".]+)\./); return m?.[1] ?? (f.id.split(".")[0] ?? ""); @@ -468,70 +943,10 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 if (nonGapFailed.length) allProblems.push(`${node}: non-gap resource failed: ${nonGapFailed.map((f) => `${f.id} (${f.error.slice(0, 60)})`).join(", ")}`); } - // ================================================================================================ - // The dry-run fixes, proved by name. - // ================================================================================================ - - // fail2ban is now HOSTABLE (capability "firewall"): what the dry-run fix buys is that a node can - // host it at all. Before, it declared the never-detected "intrusion-prevention" capability, so NO - // node could host it AND its un-hostable assignment refused the whole node's push. So the GATE is - // hostability: it must be ASSIGNED and NOT refused. - // - // Its systemd service reaching active is a SEPARATE, host-level concern this offline lab cannot - // satisfy: the VM base image ships `nftables` (so firewall's package resolves and the `firewall` - // detector is advertised — which is exactly why fail2ban is now hostable) but NOT `fail2ban`, and - // the lab segment (RFC 5737 192.0.2.0/24) has no route to the package mirror, so pacman times out - // fetching fail2ban and its deps. That is a documented LAB gap (fail2ban ∈ GAPS_NOVOX, so its - // failed `fail2ban.package` resource is tolerated like firewall's oneshot nftables.service) — it is - // reported, not gated. On an online node the package installs and the service runs. - { - const refusedF2B = refused["novox"]!.find((r) => r.name === "fail2ban"); - const assignedF2B = assigned["novox"]!.has("fail2ban"); - const active = (await on("novox", `systemctl is-active fail2ban 2>&1`)).out.trim(); - const pkg = (await on("novox", `pacman -Q fail2ban 2>&1`)).out.trim(); - report.push(`\n---- fail2ban (novox): HOSTABLE assigned=${assignedF2B} refused=${refusedF2B ? "YES" : "no"} | service=${active} package="${pkg}" ----`); - if (refusedF2B) { - allProblems.push(`fail2ban still not hostable on novox: ${refusedF2B.why}`); - } else if (!assignedF2B) { - allProblems.push(`fail2ban was not assigned to novox`); - } - if (active !== "active") { - report.push(` service not active — offline lab could not install the package (documented gap, not gated); detail:`); - report.push(` ${(await on("novox", `systemctl status fail2ban --no-pager 2>&1 | head -8`)).out}`); - } - } - - // The 7 credential sidecars: each got its fake own-secret, so each must have advanced PAST the old - // "no credential" crash (it read the delivered value). It may still fail at app-auth against the - // real app with a bogus value — that is expected and does NOT gate; only the crash being gone does. - report.push(`\n---- credential sidecars: past the "no credential" crash? (fake secret delivered) ----`); - for (const c of CREDENTIALS) { - const container = `mesh-${c.module}`; - const psMap = psMaps[c.node]!; - const status = (psMap.get(container) ?? "MISSING").split(" ")[0] ?? "MISSING"; - const delivered = credentialDelivered.get(`${c.node}/${c.module}`) ?? false; - const logs = (await on(c.node, `docker logs ${container} 2>&1 | tail -60`)).out; - const stillCrashes = logs.includes(c.crash); - const appAuth = logs.split("\n").reverse().find((l) => /fail|reject|error|401|403|refused/i.test(l) && !l.includes(c.crash))?.trim().slice(0, 90) ?? ""; - report.push(` ${c.node}/${c.module.padEnd(15)} secret=${delivered ? "delivered" : "SKIPPED"} sidecar=${status.padEnd(10)} crash("${c.crash}")=${stillCrashes ? "STILL PRESENT" : "gone"}${appAuth ? ` last:"${appAuth}"` : ""}`); - if (delivered && stillCrashes) { - allProblems.push(`${c.node}/${c.module}: credential wiring did not take — sidecar still crashes "${c.crash}"`); - } - if (!delivered && assigned[c.node]!.has(c.module)) { - allProblems.push(`${c.node}/${c.module}: fake credential was not delivered (secret accept failed)`); - } - } - const summary = report.join("\n"); console.log(summary); - // Cross-node identity proof: each node's own scoped broker accounts exist and are distinct — the - // two node-plans share one broker without colliding (both run a `postgres`, `redis`, `mssql`). - for (const acct of ["novox-postgres", "ace-postgres", "novox-redis", "ace-redis"]) { - if (!new RegExp(acct).test(users)) allProblems.push(`missing broker account ${acct}`); - } - - // Diagnostics for any CORE failure (the gaps are expected; a CORE failure is what we must see). + // Diagnostics for any CORE container that did not come up. for (const { node, mods, core } of PLAN) { const psMap = psMaps[node]!; for (const mod of mods) { @@ -544,5 +959,77 @@ test("both server sets converge together on one substrate", { skip, timeout: 3_6 } } - assert.deepEqual(allProblems, [], `the full mesh did not converge together:\n ${allProblems.join("\n ")}\n\n${summary}`); + // ================================================================================================ + // ADR 0066 — ROUTE NAMES AND THE INTERNAL CA. Additive, and deliberately only the cheap half. + // + // What is checked here is the part that is DECIDED and costs one file read: a module contributes a + // LABEL, the node carries a PUBLIC DOMAIN, and the mesh joins them — `