novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Delete the lab's registry, and bootstrap the anchor through the installer #19

Merged
jschoubben merged 14 commits from feat/bed-bootstraps-through-the-installer into main 2026-09-11 19:57:05 +00:00
Conversation 0 Commits 14 Files Changed 96
+2725 -1728
96 changed files with 2725 additions and 1728 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+13 -1
View File
@@ -144,6 +144,18 @@ export MESH_LAB_BUNDLE=<mesh-host>/examples/substrate-first-node.lock
export MESH_LAB_MODULES=<mesh-control>/examples/modules
export MESH_LAB_BUILDER=<mesh-control>/build/mesh-builder # build/, which is git-ignored
# The installer. `suite` builds it with mesh-host's `make bootstrap`, which embeds a `docker save`
# of the control-plane image — so it is built AFTER that image, in the same run, or it carries a
# stale one sealed inside a binary where nothing would ever notice. The whole-mesh bed raises its
# anchor by RUNNING this, rather than by applying a substrate bundle itself (novox/hq ADR 0067).
export MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
export MESH_LAB_CONTROL_IMAGE=mesh-control:development # optional; what it carries
# A checkout of the mesh's catalogue. The installer reads the registry's and the control plane's
# manifests from a copy of it ON THE MACHINE, because at genesis there is no forge, no build
# machine and — until the registry is up — nothing serving anything.
export MESH_LAB_CATALOG=<mesh-catalog>/modules
# Built with `go build -o <path> ./examples/<name>` in mesh-control.
export MESH_LAB_PROVISIONER=<somewhere>/postgres-provisioner
export MESH_LAB_OBJECTSTORE_PROVISIONER=<somewhere>/objectstore-provisioner
@@ -194,7 +206,7 @@ the machines instead:
```sh
incus list -c ns
incus exec <instance>-registry -- systemctl is-active docker
incus exec <instance>-anchor -- systemctl is-active docker
```
*"I cannot see progress" is not evidence of no progress.*
scenarios/a-provider.yml
+1 -3
View File
@@ -13,10 +13,8 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
images:
- postgres:17-alpine
place:
all: [runtime]
scenarios/a-public-name.yml
+1 -3
View File
@@ -17,10 +17,8 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
images:
- ghcr.io/letsencrypt/pebble:2.5.0
place:
all: [runtime]
scenarios/an-object-store.yml
+1 -6
View File
@@ -19,13 +19,8 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
images:
- minio/minio:RELEASE.2025-09-07T16-13-09Z
# The vendor's client, stocked so the provisioner has the thing it drives without reaching a
# public registry from a documentation range.
- minio/mc:RELEASE.2025-08-13T08-35-41Z
place:
all: [runtime]
scenarios/anthropic-bed.yml
+2 -4
View File
@@ -29,17 +29,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The two model-access runtimes, built by scripts/build-module-runtime.sh into the local daemon and
# stocked into the scenario's own registry, which is where the host pulls them from.
# loaded onto the machine, which holds them by their own image IDs.
- mesh-runtime-anthropic-manager:development
- mesh-runtime-anthropic-consumer:development
scenarios/audit-node.yml
+2 -3
View File
@@ -14,16 +14,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The tool runtime with the audit-logger, built by scripts/build-runtime-image.sh into the local
# daemon and stocked into the scenario's own registry, which is where the host pulls it from.
# daemon and loaded onto the machine, which holds it by its own image ID.
- mesh-runtime-audit:development
place:
scenarios/behind-nat.yml
+2 -2
View File
@@ -11,7 +11,7 @@ segments:
home:
kind: private
cidr: [192.168.1.0/24]
cidr: [10.99.1.0/24]
gateway:
to: hosting
address: [192.0.2.50] # what the world sees the household as
@@ -25,7 +25,7 @@ machines:
inbound: allow
home-server: # a dash in the name, on purpose
at: { segment: home, address: [192.168.1.135] }
at: { segment: home, address: [10.99.1.135] }
published:
- { port: 8080, on: home }
inbound: allow
scenarios/bootstrap-with-registry.yml
-23
View File
@@ -1,23 +0,0 @@
# One machine and a registry, which is the smallest scenario that can exercise a container.
#
# A sealed machine cannot reach a registry and an image placed from an archive cannot keep its
# digest (novox/hq 04-ISSUES/009), so the lab raises one inside the scenario and serves the
# images below from it. What a declaration pins is reported when this is raised — the digest
# belongs to this registry, not to the one the image came from.
scenario: bootstrap-with-registry
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
inbound: allow
images:
- alpine:3.20
place:
all: [host, runtime]
scenarios/catalogue-apps.yml
+1 -7
View File
@@ -28,6 +28,7 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
# Sized up past catalogue-small's 6GiB: this wave carries two heavy JVM/embedded-DB service
# containers (the UniFi controller and MaryTTS) on top of the substrate, mongodb, postgres and
@@ -36,14 +37,7 @@ machines:
cpus: 4
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The module server images.
- mongo:7
- lscr.io/linuxserver/unifi-controller:latest
- synesthesiam/marytts:lab
# The runtimes built by scripts/build-module-runtime.sh and stocked here. marrytts needs none.
- mesh-runtime-mongodb:development
- mesh-runtime-unifi:development
scenarios/catalogue-media.yml
+1 -6
View File
@@ -30,6 +30,7 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
# Two *arr apps (server + runtime each) on top of the first-node substrate — seven containers.
# The Servarr images are lighter than catalogue-apps' JVM pair, so catalogue-small's 6GiB is
@@ -38,13 +39,7 @@ machines:
cpus: 4
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The module server images.
- lscr.io/linuxserver/sonarr:latest
- lscr.io/linuxserver/radarr:latest
# The two runtimes built by scripts/build-module-runtime.sh and stocked here.
- mesh-runtime-sonarr:development
- mesh-runtime-radarr:development
scenarios/catalogue-mqtt.yml
+2 -9
View File
@@ -14,7 +14,7 @@
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# scripts/build-module-runtime.sh mosquitto builds mesh-runtime-mosquitto:development (carrying
# mosquitto_ctrl and the compiled bootstrap entrypoint) into the local daemon, which this scenario
# stocks and serves by digest from its own registry. eclipse-mosquitto:2 must be in the local
# pulls from the internet over its uplink. eclipse-mosquitto:2 must be in the local
# daemon to be stocked.
scenario: catalogue-mqtt
@@ -26,20 +26,13 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# mosquitto's broker (the service image) and its runtime, the latter built by
# scripts/build-module-runtime.sh mosquitto into the local daemon and stocked into the scenario's
# own registry, which is where the host pulls it from. The runtime image is reused for the
# run-once bootstrap step and for the serve container.
- eclipse-mosquitto:2
- mesh-runtime-mosquitto:development
place:
scenarios/catalogue-small.yml
+2 -7
View File
@@ -13,7 +13,7 @@
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# scripts/build-module-runtime.sh {postgres,redis,minio,plex} build the four runtime images into the
# local daemon (postgres carries psql, minio carries mc), which this scenario stocks and serves by
# digest from its own registry. The service images must be in the local daemon to be stocked.
# the internet over its uplink. Only the mesh's own images come from the local daemon.
scenario: catalogue-small
segments:
@@ -24,6 +24,7 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
# Sized up: this anchor runs the substrate (store, broker, control) plus four modules — three of
# which are a server container and a runtime container each — so a dozen containers at once. The
@@ -32,13 +33,7 @@ machines:
cpus: 4
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The module server images.
- redis:7-alpine
- minio/minio:latest
# The four per-module runtimes, built by scripts/build-module-runtime.sh and stocked here. plex
# needs no server image in the lab — its runtime serves tools with no Plex to reach.
- mesh-runtime-postgres:development
scenarios/first-node.yml
+8 -7
View File
@@ -1,8 +1,8 @@
# One machine, raising a substrate from the bundle its host carries.
#
# This is the bootstrap class (novox/hq ADR 0009): no forge, no control plane to talk to, no
# delivery. It exists to develop the steps of raising a mesh on a machine with no route out —
# a container runtime, a store, the control plane's schema in it, and the broker.
# delivery. It exists to develop the steps of raising a mesh on a machine that has nothing but a
# connection — a container runtime, a store, the control plane's schema in it, and the broker.
#
# It stops before the control plane *runs*, because there is nothing for it to serve yet.
scenario: first-node
@@ -15,14 +15,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
# Placed into a registry the scenario raises, which is what a real node pulls from anyway. The
# digests below are the ones that registry assigns, and that satisfies pinning: what is required
# is a reference that is exact and cannot move (novox/hq ADR 0006).
# The control plane's own image exists in no registry — it is built from source, and until this
# mesh has a registry of its own there is nowhere to have pushed it. So it is loaded onto the
# machine and named by the digest of its own configuration, which is exact and cannot move, which
# is what pinning asks for (novox/hq ADR 0006). The store and the broker are ordinary third-party
# images, and the machine pulls them from the internet like anything else.
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
place:
scenarios/grafana-node.yml
+1 -2
View File
@@ -14,13 +14,12 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
- mesh-runtime-grafana:development
scenarios/growing-mesh.yml
+3 -2
View File
@@ -16,17 +16,18 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
laptop:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
workstation:
at: { segment: hosting, address: [192.0.2.30] }
egress: true
inbound: allow
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
place:
scenarios/lavinmq-bed.yml
+5 -6
View File
@@ -22,8 +22,8 @@
# scripts/build-module-runtime.sh lavinmq /tmp/lavinmq.tar
# scripts/build-module-runtime.sh amqp-ping /tmp/amqp-ping.tar
# The service image cloudamqp/lavinmq:latest must be in the local daemon too — it is already stocked as
# the substrate's own broker image; each node pulls what it runs from the scenario's own registry by
# digest.
# the substrate's own broker image; each node pulls what it runs from the internet, over its own
# uplink.
scenario: lavinmq-bed
segments:
@@ -34,23 +34,22 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
laptop:
at: { segment: hosting, address: [192.0.2.11] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker (itself lavinmq), control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The lavinmq provider's runtime (reused for its run-once bootstrap and its provisioner) and the
# amqp-ping consumer's runtime, both built by scripts/build-module-runtime.sh into the local daemon
# and stocked into the scenario's own registry, which is where the hosts pull them from by digest.
# and loaded onto the machines, which hold them by their own image IDs.
# The lavinmq SERVICE image is cloudamqp/lavinmq:latest, already stocked above.
- mesh-runtime-lavinmq:development
- mesh-runtime-amqp-ping:development
scenarios/local-model-bed.yml
+1 -7
View File
@@ -24,20 +24,14 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 4GiB
cpus: 4
disk: 40GiB
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The local model server the ollama provider runs. Pulled by raise() and stocked into the scenario's
# own registry, which is where the host pulls it from. No model is pulled into it — the bed proves the
# mesh routes a consumer to the server's endpoint, not that the server generates tokens.
- ollama/ollama:latest
place:
all: [host, runtime]
scenarios/minio-node.yml
+3 -5
View File
@@ -15,17 +15,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
- minio/minio:latest
# minio's runtime, built by scripts/build-module-runtime.sh minio (it carries mc), stocked into the
# scenario's own registry.
# minio's runtime, built by scripts/build-module-runtime.sh minio (it carries mc), loaded onto
# the machine.
- mesh-runtime-minio:development
place:
scenarios/model-usage-bed.yml
+3 -5
View File
@@ -25,25 +25,23 @@ machines:
# The substrate ONLY: store, broker, control — three containers.
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 4GiB
cpus: 4
# The postgres PROVIDER (server + broker-bound runtime) and the model-usage CONSUMER (its run-once
# migrate and its long-lived event runtime). The 5432-vs-substrate conflict is gone because the
# substrate store is on the OTHER node. The runtime images plus postgres:17-alpine are pulled from
# the scenario's own registry by digest; forty gigabytes holds them with room to spare.
# the internet over the uplink; forty gigabytes holds them with room to spare.
laptop:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 4GiB
cpus: 4
disk: 40GiB
images:
# The first-node substrate: store, broker, control. postgres:17-alpine doubles as postgres's own
# service image.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The per-module runtimes, built by scripts/build-module-runtime.sh and stocked here. Each carries
# its module's code — postgres its provisioner, model-usage its consumer, tools and run-once migrate.
scenarios/openai-bed.yml
+2 -4
View File
@@ -24,17 +24,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The static-key consumer runtime, built by scripts/build-module-runtime.sh into the local daemon and
# stocked into the scenario's own registry, which is where the host pulls it from.
# loaded onto the machine, which holds it by its own image ID.
- mesh-runtime-openai-consumer:development
place:
scenarios/plex-node.yml
+2 -3
View File
@@ -15,16 +15,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# Plex's tool runtime, built by scripts/build-module-runtime.sh plex into the local daemon and
# stocked into the scenario's own registry, which is where the host pulls it from.
# loaded onto the machine, which holds it by its own image ID.
- mesh-runtime-plex:development
place:
scenarios/postgres-node.yml
+3 -4
View File
@@ -14,16 +14,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# postgres's runtime, built by scripts/build-module-runtime.sh postgres (it carries psql), stocked
# into the scenario's own registry.
# postgres's runtime, built by scripts/build-module-runtime.sh postgres (it carries psql), loaded
# onto the machine.
- mesh-runtime-postgres:development
place:
scenarios/redis-node.yml
+2 -4
View File
@@ -14,17 +14,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
- redis:7-alpine
# Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local
# daemon and stocked into the scenario's own registry, which is where the host pulls it from.
# daemon and loaded onto the machine, which holds it by its own image ID.
- mesh-runtime-redis:development
place:
scenarios/route-forwarding.yml
+2 -5
View File
@@ -29,21 +29,18 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The route-proxy's image, built from the canonical Go proxy in mesh-control by
# scripts/build-route-proxy-image.sh, and hello-web's backend, a bare alpine nc loop.
- mesh-route-proxy:development
- alpine:latest
place:
# Only the host — neither module carries a mesh-runtime. Both service images are served by the
# scenario's registry and pulled by the host, not placed inside the machine.
# internet and pulled by the host over its uplink, not placed inside the machine.
all: [host]
scenarios/schedule-tick.yml
+5 -11
View File
@@ -15,8 +15,8 @@
# - it fires AGAIN on the following minute — recurrence, not a one-shot.
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_MODULES=.../mesh-control/examples/modules
# alpine:latest must be in the local daemon; the scenario stocks it into its own registry and
# serves it by digest, which is what the scheduled container declares (via pinned("alpine")).
# alpine:latest must be in the local daemon; the machine pulls it from the internet over its
# uplink, and the scheduled container declares it exactly as the catalogue writes it.
# There is no runtime image: schedtest carries no code of its own — the scheduled container is a
# bare alpine that runs `date >> /data/runs.log` and exits.
scenario: schedule-tick
@@ -29,21 +29,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The tick container's image. schedtest has no runtime of its own — its scheduled container is a
# bare alpine that appends a timestamp and exits. alpine:latest must be in the local daemon; the
# scenario stocks it and serves it by digest, which is what the manifest pins via pinned("alpine").
- alpine:latest
place:
# Only the host — schedtest has no mesh-runtime to place. The tick image is served by the
# scenario's registry and pulled by the host, not placed inside the machine.
# Only the host — schedtest has no mesh-runtime to place. The tick image is pulled from the
# internet by the host over its uplink, not placed inside the machine.
all: [host]
scenarios/segmented-and-unforwardable.yml
+2 -2
View File
@@ -15,7 +15,7 @@ segments:
home:
kind: private
cidr: [192.168.1.0/24]
cidr: [10.99.1.0/24]
gateway:
to: hosting
address: [192.0.2.50]
@@ -53,7 +53,7 @@ machines:
inbound: allow
home-server:
at: { segment: home, address: [192.168.1.135] }
at: { segment: home, address: [10.99.1.135] }
inbound: allow
thermostat:
scenarios/sonarr-node.yml
+1 -2
View File
@@ -14,13 +14,12 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
- mesh-runtime-sonarr:development
scenarios/the-ordinary-shape.yml
+4 -4
View File
@@ -21,7 +21,7 @@ segments:
home:
kind: private
cidr: [192.168.1.0/24, "2001:db8:b:1::/64"]
cidr: [10.99.1.0/24, "2001:db8:b:1::/64"]
mtu: 1492
gateway:
to: isp-home
@@ -61,17 +61,17 @@ machines:
inbound: allow
home-server:
at: { segment: home, address: [192.168.1.135, "2001:db8:b:1::135"] }
at: { segment: home, address: [10.99.1.135, "2001:db8:b:1::135"] }
published:
- { port: 443, on: home }
inbound: allow
workstation:
at: { segment: home, address: [192.168.1.250, "2001:db8:b:1::250"] }
at: { segment: home, address: [10.99.1.250, "2001:db8:b:1::250"] }
inbound: deny
laptop:
at: { segment: home, address: [192.168.1.98, "2001:db8:b:1::98"] }
at: { segment: home, address: [10.99.1.98, "2001:db8:b:1::98"] }
inbound: deny
# No `place:` yet. The node host it would place does not exist — this lab is being built to
scenarios/tools-confluence.yml
+3 -5
View File
@@ -11,7 +11,7 @@
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# scripts/build-module-runtime.sh confluence builds mesh-runtime-confluence:development into the
# local daemon, which this scenario stocks and serves by digest from its own registry. confluence
# local daemon, which the machine pulls from the internet over its uplink. confluence
# needs no service image — it is tools-only.
scenario: tools-confluence
@@ -23,17 +23,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# confluence's runtime, built by scripts/build-module-runtime.sh confluence into the local daemon
# and stocked into the scenario's own registry, which is where the host pulls it from. There is no
# and loaded onto the machine, which holds it by its own image ID. There is no
# service image: confluence is tools-only and outbound-only.
- mesh-runtime-confluence:development
scenarios/tools-gitlab.yml
+3 -5
View File
@@ -12,7 +12,7 @@
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# scripts/build-module-runtime.sh gitlab builds mesh-runtime-gitlab:development into the local
# daemon, which this scenario stocks and serves by digest from its own registry. gitlab needs no
# daemon, which the machine pulls from the internet over its uplink. gitlab needs no
# service image — it is tools-only.
scenario: tools-gitlab
@@ -24,17 +24,15 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
images:
# The first-node substrate: store, broker, control.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# gitlab's runtime, built by scripts/build-module-runtime.sh gitlab into the local daemon and
# stocked into the scenario's own registry, which is where the host pulls it from. There is no
# loaded onto the machine, which holds it by its own image ID. There is no
# service image: gitlab is tools-only and outbound-only.
- mesh-runtime-gitlab:development
scenarios/two-node-db.yml
+3 -9
View File
@@ -21,6 +21,7 @@ machines:
# containers on a node, which this one never does.
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 4GiB
cpus: 4
@@ -31,25 +32,18 @@ machines:
# so — and convergence would present as "the mesh hangs". Six gigabytes gives it room.
laptop:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 6GiB
cpus: 4
# The runtime images (280MB–600MB each) plus the service images — two of them heavy app images
# (Baserow ~1.5GB, Letta ~1.8GB) — are pulled from the scenario's own registry by digest, so the
# (Baserow ~1.5GB, Letta ~1.8GB) — are pulled from the internet over the uplink, so the
# same bytes land on this node twice. The pool default root disk exhausts mid-apply ("no space
# left on device"); sixty gigabytes holds the whole chain.
disk: 60GiB
images:
# The first-node substrate: store, broker, control. postgres:17-alpine doubles as postgres's own
# service image.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The module service images.
- redis:7-alpine
- baserow/baserow:latest
- letta/letta:latest
# The per-module runtimes, built by scripts/build-module-runtime.sh and stocked here. Each carries
# its module's provisioner, so no separate mesh-provision-* image is listed — the runtime is the
# provisioner (ADR 0048).
scenarios/two-nodes.yml
+2 -17
View File
@@ -15,6 +15,7 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
# The whole substrate, the registry, the builder, an adopted workload and the modules under
# test all land here — eleven containers before the forge arrives. At the 1GiB default this
@@ -24,21 +25,12 @@ machines:
cpus: 4
laptop:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 2GiB
images:
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# So a module can mirror one into a registry of the mesh's own. The scenario's registry serves
# what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved
# the same way.
- registry:2
# A real third-party workload, for adopting one the way the conversion will. Its database is
# the substrate's postgres image rather than its own: what is under test is the mesh delivering
# a module, not which postgres it delivers.
- ghcr.io/umami-software/umami:postgresql-latest
# And the builder, because it is a module the mesh assigns rather than a program somebody
# starts by hand — which is the only way its credential can be one the mesh delivered.
- mesh-builder:development
@@ -47,17 +39,10 @@ images:
- mesh-provision-postgres:development
# And the proxy, which is what turns a route grant into traffic actually arriving.
- mesh-route-proxy:development
# And the cache, with its provisioner — the third provision after a database and a bucket,
# and the first whose tenancy is a keyspace rather than a namespace something else enforces.
- redis:7-alpine
- mesh-provision-redis:development
# And the object store's provisioner, so the module describing it can be planned. Without it
# that module still names an image nothing serves, and planning it is refused — correctly.
- mesh-provision-objectstore:development
# And a forge, so one of the real module descriptions can be started rather than only planned.
# It is the first of them to run: it needs a database from another module, a credential it did
# not choose, and a connection string it could not have written itself.
- gitea/gitea:1.22
place:
all: [host, runtime]
scenarios/whole-mesh-ace.yml
+36 -34
View File
@@ -10,11 +10,11 @@
# the mesh confirms the paths exist and mounts them, but creates and chowns none of it.
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# The runtimes are built by scripts/build-module-runtime.sh (one per module); every server image must
# be in the local daemon to be stocked. The media/app images are pulled by their pinned digests and
# tagged :mesh so repositoryFor matches the module.json paths (postgres/redis/portainer/mssql reuse
# their existing local tags). The test loads each committed module.json from mesh-catalog and rewrites
# its image references to what this scenario's own registry serves by digest.
# The runtimes are built by scripts/build-module-runtime.sh (one per module) and must be in the local
# daemon, because nothing serves them and nothing can. Every media/app image is pulled from the
# internet by the node itself, over its uplink, by the digest its module.json already pins. The test
# loads each committed module.json from mesh-catalog and rewrites only OUR image references, to the
# ID the machine holds each one under.
scenario: whole-mesh-ace
segments:
@@ -25,50 +25,52 @@ segments:
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 4GiB
cpus: 4
disk: 20GiB
# The substrate only, so the control plane's image only. The runtimes belong on the node that
# runs the modules, and a 20GiB disk has no room for them anyway.
images: [mesh-control:development]
# The whole ace service set — 24 modules, ~50 containers, several heavy (Plex, Home Assistant,
# Letta ~1.8GiB, Baserow ~1.5GiB, the UniFi controller's JVM, mssql ~2GiB). Sized past novox.
ace:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 18GiB
cpus: 8
disk: 120GiB
# Every runtime. Not mesh-control: the control plane runs on the anchor.
images:
- mesh-runtime-postgres:development
- mesh-runtime-redis:development
- mesh-runtime-mssql:development
- mesh-runtime-portainer:development
- mesh-runtime-sonarr:development
- mesh-runtime-radarr:development
- mesh-runtime-lidarr:development
- mesh-runtime-plex:development
- mesh-runtime-bazarr:development
- mesh-runtime-nzbget:development
- mesh-runtime-qbittorrent:development
- mesh-runtime-jackett:development
- mesh-runtime-ombi:development
- mesh-runtime-tautulli:development
- mesh-runtime-bookshelf:development
- mesh-runtime-home-assistant:development
- mesh-runtime-mosquitto:development
- mesh-runtime-influxdb:development
- mesh-runtime-grafana:development
- mesh-runtime-baserow:development
- mesh-runtime-letta:development
- mesh-runtime-nodered:development
- mesh-runtime-searxng:development
- mesh-runtime-unifi:development
images:
# The first-node substrate.
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The module server images. Reused local tags where the exact version does not matter for a boot
# (postgres/redis/portainer/mssql); pinned-digest :mesh tags for the media/app images.
- redis:7-alpine
- lscr.io/linuxserver/sonarr:mesh
- lscr.io/linuxserver/radarr:mesh
- lscr.io/linuxserver/lidarr:mesh
- lscr.io/linuxserver/bazarr:mesh
- lscr.io/linuxserver/nzbget:mesh
- lscr.io/linuxserver/qbittorrent:mesh
- lscr.io/linuxserver/jackett:mesh
- lscr.io/linuxserver/ombi:mesh
- lscr.io/linuxserver/tautulli:mesh
- lscr.io/linuxserver/unifi-controller:mesh
- plexinc/pms-docker:mesh
- ghcr.io/pennydreadful/bookshelf:mesh
- ghcr.io/home-assistant/home-assistant:mesh
- eclipse-mosquitto:mesh
- influxdb:mesh
- grafana/grafana:mesh
- baserow/baserow:mesh
- letta/letta:mesh
- nodered/node-red:mesh
- searxng/searxng:mesh
- valkey/valkey:mesh
- portainer/portainer-ce:latest
- mcr.microsoft.com/mssql/server:2022-latest
# The per-module runtimes (built by scripts/build-module-runtime.sh).
- mesh-runtime-postgres:development
- mesh-runtime-redis:development
scenarios/whole-mesh-full.yml
+160 -71
View File
@@ -1,95 +1,184 @@
# The FULL mesh: both server sets on ONE substrate, converging together — the final stage of the
# whole-mesh rehearsal (novox/hq). Combines scenarios/whole-mesh-novox.yml and whole-mesh-ace.yml.
# The FULL mesh in its REAL production shape: two segments, one access point, one overlay.
#
# anchor — substrate ONLY (store, broker, control).
# novox — the 17-module novox set (providers + web apps + route-proxy + mailu + firewall).
# ace — the 24-module ace set (media/home stack), its /services/media library pre-created.
# This is the first multi-segment whole-mesh bed. The earlier flat whole-mesh-full sat every node
# on one public segment with a SEPARATE `anchor` carrying the substrate. Production is not flat, and
# there is no separate anchor: `novox` IS the anchor. It sits on the routable `hosting` segment,
# runs the substrate (store, broker, control) AND its own service set AND is the overlay hub and the
# public ingress. `ace`, `shanks` and `g14` sit on the household `home` segment BEHIND a NAT gateway
# — the access point — reachable from the outside only through what they dial out to.
#
# An overlay is placed across all three so cross-node `at` resolves. Each service node is
# self-contained (its own postgres/redis), so nothing crosses a node boundary except enrolment and
# the shared broker/store on anchor — which is exactly what this stage proves converges for two
# independent node-plans at once on one substrate.
# hosting (public, routable) home (private, behind the access point)
# novox 192.0.2.20 ── anchor ace 10.99.1.10 home server, media/IoT set
# substrate + novox set shanks 10.99.1.20 workstation (light)
# overlay hub, ingress g14 10.99.1.30 workstation (light)
#
# The `home` gateway masquerades v4 outbound and forwards inbound (an ordinary household router).
# Home nodes reach novox's public 192.0.2.20 by dialling OUT through it: the substrate broker (5671),
# the mesh's own artifact store, and — the thing this bed exists to prove — the WireGuard overlay hub
# (51820/udp). The hub keepalive holds the NAT hole open so the tunnel, once formed, stays up. novox
# cannot initiate to a home node at all; every home↔novox path is either the overlay or a forwarded
# port.
#
# EVERY NODE HAS EGRESS, which is not a convenience. Third-party images — postgres, the whole Mailu
# stack, Plex, everything the modules actually run — are pulled from the internet, because that is
# where a real node gets them. The lab used to serve them from a registry it raised inside the
# scenario; no production mesh has one, so a bootstrap that could only work against it went green
# here and would have failed anywhere else. What is loaded onto a machine now is only what exists in
# no registry at all: the mesh's own images, named per machine below.
#
# Egress is a SECOND path, not a replacement for the topology. Each node still reaches the rest of
# the scenario through its declared gateway — that is where the overlay handshake has to survive a
# masquerade — and the uplink carries only what leaves the scenario entirely.
#
# THE UNPROVEN THING (what the flat beds never tested): does the overlay tunnel FORM across the
# access point — a home node dialling novox's public hub endpoint, the handshake completing through
# the gateway's masquerade? The driving test verifies the WireGuard handshake and cross-segment
# reachability over the overlay explicitly, and reports form-vs-break as its headline.
#
# Substrate-on-novox collides on two host ports the separate-anchor beds never hit: the substrate
# store binds 127.0.0.1:5432 and novox's postgres provider publishes 5432; the substrate broker binds
# 5671 + 127.0.0.1:5672 and novox's lavinmq provider publishes 5672. The driving test REMAPS those two
# provider host publishes off the substrate's ports (consumers reach the providers over the mesh
# network on the container port, so the host side is free to move). Reported as a topology finding.
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# The images are the UNION of the two per-server scenarios; every one is already built/pulled by the
# per-server bed prerequisites (scripts/build-module-runtime.sh, build-route-proxy-image.sh, the
# mailu/keycloak and media :mesh digest pulls).
# MESH_LAB_BOOTSTRAP_BINARY=.../mesh-bootstrap MESH_LAB_CATALOG=.../mesh-catalog/modules
#
# GENESIS AND JOINING ARE TWO DIFFERENT ACTS, and this bed distinguishes them. novox is brought
# into existence by `mesh-bootstrap` — the same program a bare machine runs — and is afterwards a
# working mesh of one, with a registry and a control plane that is an ordinary module pinned to an
# image that registry serves. ace, shanks and g14 then JOIN it: host binary, token, enrol, run. No
# bootstrap, no substrate, no registry. novox is never enrolled twice, because the installer
# already did it.
#
# The images: are the UNION of the novox set (feat/novox-conversions @ 431310f: the slug + roundcube
# fixes, so only-office/de-spiegel/amqp-email-forwarder now resolve) and the ace media/home set, and
# every one of them must already be BUILT on the workstation — nothing can fetch them.
scenario: whole-mesh-full
segments:
# The routable segment. novox lives here, and the overlay hub endpoint is a public address here.
hosting:
kind: public
cidr: [192.0.2.0/24]
# The household segment behind the access point. Its gateway is an ordinary home router: it
# masquerades v4 outbound, forwards inbound, and expires idle mappings after two minutes — which
# is exactly the NAT hole a WireGuard keepalive has to hold open.
home:
kind: private
cidr: [10.99.1.0/24]
gateway:
to: hosting
address: [192.0.2.50] # what the world sees the household as
nat: [v4]
forwardable: true
mapping_ttl: 120s
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
inbound: allow
memory: 4GiB
cpus: 4
disk: 20GiB
# The anchor: substrate (store, broker, control) + the whole novox service set + overlay hub +
# public ingress. Bigger than the flat bed's novox, because it now carries the substrate too.
novox:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 16GiB
cpus: 6
disk: 100GiB
memory: 24GiB
cpus: 8
disk: 130GiB
# The proxy, and a runtime per module novox is assigned. step-ca, photos, invoicing, novox.be,
# only-office, de-spiegel, amqp-email-forwarder, registry, firewall and fail2ban are not here
# because they carry no runtime image of their own — what they run is third-party or is the
# node itself.
#
# **mesh-control is NOT here, and its absence is the point** (novox/hq ADR 0067). The anchor is
# brought into existence by the installer, and the installer carries the control plane's image
# inside itself — that is the whole reason a machine that can reach no registry can still raise
# a mesh. Handing it over from the workstation as well would mean the bed never found out
# whether the installer really carries it: the load would say "already held" and the fiction
# would be invisible, which is exactly the class of thing the lab's own registry used to hide.
images:
- mesh-route-proxy:development
- mesh-runtime-postgres:development
- mesh-runtime-redis:development
- mesh-runtime-minio:development
- mesh-runtime-mongodb:development
- mesh-runtime-mssql:development
- mesh-runtime-lavinmq:development
- mesh-runtime-keycloak:development
- mesh-runtime-gitea:development
- mesh-runtime-nextcloud:development
- mesh-runtime-umami:development
- mesh-runtime-verdaccio:development
- mesh-runtime-portainer:development
- mesh-runtime-mailu:development
# The home server: the whole ace media/home set — 24 modules, ~50 containers, several heavy
# (Plex, Home Assistant, Letta, Baserow, the UniFi JVM, mssql). Behind the gateway.
ace:
at: { segment: hosting, address: [192.0.2.30] }
at: { segment: home, address: [10.99.1.10] }
egress: true
inbound: allow
memory: 18GiB
cpus: 6
disk: 120GiB
# A runtime per module ace is assigned, and nothing of novox's. This is the point of saying it
# per machine: ace has no business holding a Keycloak runtime, and an operator's home server
# would not.
images:
- mesh-runtime-postgres:development
- mesh-runtime-redis:development
- mesh-runtime-mssql:development
- mesh-runtime-portainer:development
- mesh-runtime-sonarr:development
- mesh-runtime-radarr:development
- mesh-runtime-lidarr:development
- mesh-runtime-plex:development
- mesh-runtime-bazarr:development
- mesh-runtime-nzbget:development
- mesh-runtime-qbittorrent:development
- mesh-runtime-jackett:development
- mesh-runtime-ombi:development
- mesh-runtime-tautulli:development
- mesh-runtime-bookshelf:development
- mesh-runtime-home-assistant:development
- mesh-runtime-mosquitto:development
- mesh-runtime-influxdb:development
- mesh-runtime-grafana:development
- mesh-runtime-baserow:development
- mesh-runtime-letta:development
- mesh-runtime-nodered:development
- mesh-runtime-searxng:development
- mesh-runtime-unifi:development
# Two workstations on the same home LAN. Light on purpose: they enrol, join the overlay, and run
# one small module (portainer) so a real module converges on each without heavy load. Same-LAN
# nodes with no overlay endpoint of their own hairpin the hub rather than peering directly, which
# is the normal case and is fine.
shanks:
at: { segment: home, address: [10.99.1.20] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
disk: 30GiB
# One module, one runtime. Handing these two the whole union would put roughly thirty gigabytes
# of images onto a thirty-gigabyte disk, which is how you learn that "the lab loads everything
# everywhere" was never a description of anything real.
images: [mesh-runtime-portainer:development]
g14:
at: { segment: home, address: [10.99.1.30] }
egress: true
inbound: allow
memory: 3GiB
cpus: 2
disk: 30GiB
images: [mesh-runtime-portainer:development]
# The union of what the machines above ask for. This is the list the workstation must be able to
# export — every entry is one of the mesh's own images, built from source and published nowhere, so
# a machine holds it because it was handed it. Everything else the modules run comes from the
# internet and is not named here at all.
images:
# --- substrate + shared ---
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
- redis:7-alpine
- portainer/portainer-ce:latest
- mcr.microsoft.com/mssql/server:2022-latest
# --- novox server images ---
- minio/minio:latest
- mongo:7
- quay.io/keycloak/keycloak:mesh
- gitea/gitea:1.22
- nextcloud:stable
- ghcr.io/umami-software/umami:postgresql-latest
- alpine:latest
- verdaccio/verdaccio:6
- registry:2
- registry-api.novox.be/novox/invoicing-app:latest
- registry-api.novox.be/novox/invoicing-api:latest
- ghcr.io/mailu/unbound:mesh
- ghcr.io/mailu/admin:mesh
- ghcr.io/mailu/dovecot:mesh
- ghcr.io/mailu/postfix:mesh
- ghcr.io/mailu/rspamd:mesh
- ghcr.io/mailu/webmail:mesh
- ghcr.io/mailu/nginx:mesh
# --- ace server images ---
- lscr.io/linuxserver/sonarr:mesh
- lscr.io/linuxserver/radarr:mesh
- lscr.io/linuxserver/lidarr:mesh
- lscr.io/linuxserver/bazarr:mesh
- lscr.io/linuxserver/nzbget:mesh
- lscr.io/linuxserver/qbittorrent:mesh
- lscr.io/linuxserver/jackett:mesh
- lscr.io/linuxserver/ombi:mesh
- lscr.io/linuxserver/tautulli:mesh
- lscr.io/linuxserver/unifi-controller:mesh
- plexinc/pms-docker:mesh
- ghcr.io/pennydreadful/bookshelf:mesh
- ghcr.io/home-assistant/home-assistant:mesh
- eclipse-mosquitto:mesh
- influxdb:mesh
- grafana/grafana:mesh
- baserow/baserow:mesh
- letta/letta:mesh
- nodered/node-red:mesh
- searxng/searxng:mesh
- valkey/valkey:mesh
# --- per-module runtimes (union) ---
- mesh-runtime-postgres:development
- mesh-runtime-redis:development
@@ -97,11 +186,11 @@ images:
- mesh-runtime-portainer:development
- mesh-runtime-minio:development
- mesh-runtime-mongodb:development
- mesh-runtime-lavinmq:development
- mesh-runtime-keycloak:development
- mesh-runtime-gitea:development
- mesh-runtime-nextcloud:development
- mesh-runtime-umami:development
- mesh-runtime-photos:development
- mesh-runtime-verdaccio:development
- mesh-runtime-mailu:development
- mesh-route-proxy:development
scenarios/whole-mesh-novox.yml
+26 -32
View File
@@ -17,9 +17,10 @@
#
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
# The runtimes are built by scripts/build-module-runtime.sh (one per module that has code) and the
# route-proxy image by scripts/build-route-proxy-image.sh; every server image must be in the local
# daemon to be stocked. The test loads each committed module.json from mesh-catalog and rewrites its
# image references to what this scenario's own registry serves by digest.
# route-proxy image by scripts/build-route-proxy-image.sh; those must be in the local daemon,
# because nothing serves them and nothing can. Every third-party image is pulled from the internet
# over each node's uplink. The test loads each committed module.json from mesh-catalog and rewrites
# only OUR image references, to the ID the machine holds each one under.
scenario: whole-mesh-novox
segments:
@@ -31,55 +32,48 @@ machines:
# The substrate ONLY: store, broker, control. Nothing else lands here.
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
inbound: allow
memory: 4GiB
cpus: 4
disk: 20GiB
# The substrate only, so the control plane's image only. Handing this machine the whole set of
# runtimes would fill a 20GiB disk with images nothing on it will ever start.
images: [mesh-control:development]
# The whole novox service set — ~38 containers (five providers with runtimes, six consumers with
# runtimes, portainer/verdaccio/registry/route-proxy, the nine-container Mailu stack and its
# runtime) plus two node-level modules. mssql alone wants ~2GiB; Mailu, Nextcloud and Keycloak are
# each heavy. Sized well past the two-node-db bed's second node.
novox:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
inbound: allow
memory: 16GiB
cpus: 8
# ~14GiB of images are pulled from the scenario's own registry by digest, several of them large
# ~14GiB of images are pulled from the internet over the uplink, several of them large
# (mssql 1.7GiB, invoicing-api 1.9GiB, nextcloud 1.5GiB, umami/mongo ~0.9GiB), plus writable
# layers and the runtimes. A hundred gigabytes holds the whole set without exhausting the disk
# mid-apply.
disk: 100GiB
# Every runtime, and the proxy. Not mesh-control: the control plane runs on the anchor.
images:
- mesh-runtime-postgres:development
- mesh-runtime-redis:development
- mesh-runtime-minio:development
- mesh-runtime-mongodb:development
- mesh-runtime-mssql:development
- mesh-runtime-keycloak:development
- mesh-runtime-gitea:development
- mesh-runtime-nextcloud:development
- mesh-runtime-umami:development
- mesh-runtime-photos:development
- mesh-runtime-portainer:development
- mesh-runtime-verdaccio:development
- mesh-runtime-mailu:development
- mesh-route-proxy:development
images:
# The first-node substrate: store, broker, control. postgres:17-alpine doubles as the postgres
# provider's own service image (and Mailu's internal admin DB).
- postgres:17-alpine
- cloudamqp/lavinmq:latest
- mesh-control:development
# The module server images. Each is stocked under the repository path its module.json names, so the
# test's rewrite (pinned(repositoryFor(image))) finds it.
- redis:7-alpine
- minio/minio:latest
- mongo:7
- mcr.microsoft.com/mssql/server:2022-latest
- quay.io/keycloak/keycloak:mesh
- gitea/gitea:1.22
- nextcloud:stable
- ghcr.io/umami-software/umami:postgresql-latest
- alpine:latest
- portainer/portainer-ce:latest
- verdaccio/verdaccio:6
- registry:2
- registry-api.novox.be/novox/invoicing-app:latest
- registry-api.novox.be/novox/invoicing-api:latest
# The Mailu stack (pulled by digest, tagged :mesh so repositoryFor matches the module.json paths).
- ghcr.io/mailu/unbound:mesh
- ghcr.io/mailu/admin:mesh
- ghcr.io/mailu/dovecot:mesh
- ghcr.io/mailu/postfix:mesh
- ghcr.io/mailu/rspamd:mesh
- ghcr.io/mailu/webmail:mesh
- ghcr.io/mailu/nginx:mesh
# The per-module runtimes (built by scripts/build-module-runtime.sh). registry, route-proxy,
# invoicing, firewall and fail2ban carry no mesh-runtime image; route-proxy ships its own.
- mesh-runtime-postgres:development
src/cli.ts
+6 -4
View File
@@ -233,10 +233,12 @@ async function main(): Promise<void> {
const seconds = ((Date.now() - started) / 1000).toFixed(1);
console.log(`\nraised ${raised.instanceId} in ${seconds}s — ${raised.machines.length} machines usable`);
if (raised.images.length > 0) {
// Printed because this is what a declaration pins, and it is not knowable until the
// scenario has been raised — the digest belongs to this registry.
console.log(`\nimages served, pinned by digest:`);
for (const image of raised.images) console.log(` ${image}`);
// Printed because this is what a declaration names them by, and it is not knowable until
// the image has been built — an image ID is the digest of its own configuration.
console.log(`\nthe mesh's own images, as the machines now hold them:`);
for (const image of raised.images) {
console.log(` ${image.requested} → ${image.reference}`);
}
}
if (scenario.snapshot) {
const took = await snapshot(raised.instanceId, scenario.snapshot);
src/declaration/parse.ts
+3
View File
@@ -41,6 +41,9 @@ function normaliseMachine(raw: unknown): Machine {
if (machine["memory"] !== undefined) result.memory = String(machine["memory"]);
if (machine["cpus"] !== undefined) result.cpus = Number(machine["cpus"]);
if (machine["disk"] !== undefined) result.disk = String(machine["disk"]);
// Absent and empty are different: absent means "all of the scenario's images", an explicit empty
// list means "none". A machine that runs nothing of ours should be able to say so.
if (machine["images"] !== undefined) result.images = toList(machine["images"]);
return result;
}
src/declaration/types.ts
+25 -4
View File
@@ -110,6 +110,19 @@ export interface Machine {
*/
memory?: string;
cpus?: number;
/**
* Which of the scenario's `images:` this machine is handed.
*
* Absent means all of them, which is right for a one-machine bed and wrong for a mesh: a
* workstation running one small module does not want forty runtimes copied onto a 30GiB disk.
* That is not a lab economy, it is what is true — an operator's machine holds the images its own
* modules need, because somebody put them there.
*
* Every entry must appear in the scenario's `images:`. Naming one that does not is refused
* rather than ignored, because a machine silently missing an image fails much later, inside an
* apply, as a container that will not start.
*/
images?: string[];
/**
* Root disk size, e.g. "60GiB". Left unset, the VM uses the storage pool's default, which is
* enough for a handful of modules. A broad install that stocks many runtime + service images
@@ -142,11 +155,19 @@ export interface Scenario {
policy?: Policy[];
place?: Placement;
/**
* Container images this scenario needs inside it.
* **The mesh's own images** — the ones that exist in no registry and are put onto a machine by
* whoever built them.
*
* A sealed machine cannot reach a registry, so the lab raises one on a public segment and
* serves these from it. Written as tags — the digest a declaration pins is the one THIS
* registry assigns, and it is reported when the scenario is raised.
* mesh-control, mesh-builder, mesh-route-proxy, the per-module runtimes and the provisioners are
* built from source and published nowhere. A machine gets them the way an operator's machine
* does: they are built on the workstation, loaded onto the machine, and named by the digest of
* their own image configuration. Written as tags, because a tag is what `docker save` can
* export; what a declaration then pins is the image ID, reported when the scenario is raised.
*
* **Third-party images do not belong here.** postgres, gitea, the mailu stack and everything
* else are pulled from the internet over a machine's `egress` uplink, exactly as they are in
* production. The lab used to serve them from a registry of its own, and that registry did not
* exist anywhere else — so every bootstrap problem it papered over went unfound.
*/
images?: string[];
/** Name the state once placement finishes, so a run can return to it. */
src/declaration/validate.ts
+22 -12
View File
@@ -15,6 +15,7 @@
import type { Scenario, Segment } from "./types.ts";
import { contains, familyOf, parseAddress, parseCidr, type Cidr } from "./net.ts";
import { mustBeHandedOver } from "../pinning.ts";
/** RFC 5737 and RFC 3849. The only addresses guaranteed never to route on the real internet. */
const DOCUMENTATION_RANGES = [
@@ -315,29 +316,38 @@ export function validate(scenario: Scenario): void {
}
}
for (const image of scenario.images ?? []) {
const declaredImages = scenario.images ?? [];
for (const image of declaredImages) {
if (!image.trim()) {
problems.push("images: an empty entry names nothing");
} else if (image.includes("@sha256:")) {
// The digest a declaration pins is the one the LAB's registry assigns, which is not
// knowable before the scenario is raised. Naming an upstream digest here would pin
// something this registry will never serve.
// A tag, because a tag is what `docker save` exports. The reference a declaration ends up
// using is the image's own ID, which is not knowable until the image has been built.
problems.push(
`images: '${image}' is pinned by digest. Name it by tag — the lab's registry assigns ` +
`its own digest and reports it when the scenario is raised`,
`images: '${image}' is pinned by digest. Name it by tag — what a declaration uses is the ` +
`ID of the image loaded onto the machine, reported when the scenario is raised`,
);
} else if (!mustBeHandedOver(image)) {
// **The rule that replaced the lab's registry.** Anything with somewhere to be fetched from
// is fetched from there, over the machine's uplink, exactly as in production. Serving it
// from inside the scenario instead is what hid the bootstrap faults this lab exists to find.
problems.push(
`images: '${image}' is not one of the mesh's own images, so nothing loads it. It is ` +
`pulled from the internet by the machine that needs it — give that machine 'egress: true' ` +
`and delete this line. Only mesh-* images, which exist in no registry, are placed by hand`,
);
}
}
if ((scenario.images ?? []).length > 0) {
const hasPublicV4 = Object.values(scenario.segments)
.some((s) => s.kind === "public" && s.cidr.some((c) => !c.includes(":")));
if (!hasPublicV4) {
for (const [name, machine] of Object.entries(scenario.machines)) {
for (const image of machine.images ?? []) {
if (!declaredImages.includes(image)) {
problems.push(
"images: this scenario declares images and has no public IPv4 segment to serve them " +
"from. The registry stands in for the outside world, so it sits on a public segment",
`machines.${name}.images: '${image}' is not in this scenario's images:. A machine can ` +
`only be handed one of the images the scenario says it has`,
);
}
}
}
if (problems.length > 0) throw new DeclarationError(problems);
}
src/lifecycle/address.ts
+119
View File
@@ -180,6 +180,20 @@ function withPrefix(scenario: Scenario, segment: string, address: string): strin
*
* The router's inside address is the first host address of the range, chosen rather than
* declared because a scenario has nothing to say about it.
*
* **A machine with `egress` is routed differently, and it has to be.** The scenery inside a
* scenario — a gateway container, the transit router — reaches the scenario and nothing else: it
* has no route to the real internet, and never will, because it exists to reproduce a household
* router rather than to be one. So a default route pointing at it is a black hole for anything
* outside, and it wins over the uplink's DHCP route on metric. A machine that must pull an image
* would then sit there failing, with a default route that looks perfectly reasonable.
*
* So an egress machine keeps the uplink as its default and gets an EXPLICIT route to every other
* segment in the scenario, through the same gateway or transit it would otherwise have defaulted
* to. Where there is no such path, the range is made `unreachable` rather than left to fall
* through: 192.168.1.0/24 in a scenario is a documentation range in spirit but an ordinary private
* one in fact, and letting it escape to the uplink would put scenario traffic on whatever network
* the workstation happens to sit on.
*/
export async function applyDefaultRoutes(
scenario: Scenario,
@@ -195,6 +209,13 @@ export async function applyDefaultRoutes(
// segment routes through transit instead — otherwise it can reach its own network and
// nothing else, which is not what being on the internet means.
const behind = spec.at.find((a) => scenario.segments[a.segment]?.gateway);
if (spec.egress) {
await routeScenarioExplicitly(scenario, machine, name);
log(` routed ${machine} inside the scenario, its default out through the uplink`);
continue;
}
if (!behind) {
await routeViaTransit(scenario, spec, name);
continue;
@@ -219,6 +240,104 @@ export async function applyDefaultRoutes(
}
}
/** One route a machine with egress needs, so the scenario stays reachable and stays inside. */
export interface ScenarioRoute {
/** The range this route is for. */
cidr: string;
/** The next hop inside the scenario, or null when there is none and the range is unreachable. */
via: string | null;
}
/**
* The routes a machine with egress needs into the rest of the scenario.
*
* Pure, and exported, because this is the decision that keeps the uplink and the declared gateway
* from fighting — and a decision only a full raise could check is one nobody checks.
*
* One route per segment the machine is not already on, through whatever it would have defaulted to:
* its gateway if it sits behind one, transit if it sits on a public segment and transit exists.
* What has no such path is `unreachable` — the faithful translation of the state it was in before,
* where its default route pointed into scenery that dropped it, and safer, because an unreachable
* route cannot be answered by whatever network the workstation happens to sit on.
*/
export function scenarioRoutesFor(scenario: Scenario, machine: string): ScenarioRoute[] {
const spec = scenario.machines[machine];
if (!spec || spec.at === "detached") return [];
const at = spec.at;
const onSegments = new Set(at.map((a) => a.segment));
const behindGateway = at.some((a) => scenario.segments[a.segment]?.gateway);
const routes: ScenarioRoute[] = [];
for (const [segment, segmentSpec] of Object.entries(scenario.segments)) {
if (onSegments.has(segment)) continue;
for (const cidr of segmentSpec.cidr) {
const slash = cidr.lastIndexOf("/");
if (slash === -1) continue;
const v6 = cidr.slice(0, slash).includes(":");
routes.push({
cidr,
via: behindGateway ? gatewayInside(scenario, at, v6) : transitOn(scenario, at, v6),
});
}
}
return routes;
}
/**
* Apply those routes, and leave the default to the uplink.
*
* No `dev`: every next hop here is on-link, so the kernel picks the interface, and asking `awk` to
* count links is one more thing that can pick `docker0`.
*/
async function routeScenarioExplicitly(
scenario: Scenario,
machine: string,
name: string,
): Promise<void> {
for (const { cidr, via } of scenarioRoutesFor(scenario, machine)) {
const family = cidr.slice(0, cidr.lastIndexOf("/")).includes(":") ? "-6" : "-4";
const route = via ? `${cidr} via ${via}` : `unreachable ${cidr}`;
await incus(
["exec", name, "--", "sh", "-c", `ip ${family} route replace ${route} 2>/dev/null || true`],
30_000,
);
}
}
/** The inside address of the gateway this machine sits behind: the first host address. */
function gatewayInside(scenario: Scenario, at: Attachment[], v6: boolean): string | null {
const behind = at.find((a) => scenario.segments[a.segment]?.gateway);
if (!behind) return null;
for (const range of scenario.segments[behind.segment]?.cidr ?? []) {
const slash = range.lastIndexOf("/");
if (slash === -1) continue;
const base = range.slice(0, slash);
if (base.includes(":") !== v6) continue;
if (v6) return `${base.replace(/::$/, "")}::1`;
const octets = base.split(".");
octets[3] = "1";
return octets.join(".");
}
return null;
}
/** The transit router's address on the public segment this machine sits on. */
function transitOn(scenario: Scenario, at: Attachment[], v6: boolean): string | null {
// One public segment means everything public is adjacent and no transit router is raised, so
// there is nothing to point at — see raiseTransit.
const publicSegments = Object.values(scenario.segments).filter((s) => s.kind === "public");
if (publicSegments.length < 2) return null;
const onPublic = at.find((a) => scenario.segments[a.segment]?.kind === "public");
if (!onPublic) return null;
for (const cidr of scenario.segments[onPublic.segment]?.cidr ?? []) {
if (cidr.slice(0, cidr.lastIndexOf("/")).includes(":") !== v6) continue;
const via = transitAddress(cidr);
if (via) return via.slice(0, via.lastIndexOf("/"));
}
return null;
}
/** A machine on a public segment reaches the other public networks through transit. */
async function routeViaTransit(
scenario: Scenario,
src/lifecycle/base.ts
+14 -7
View File
@@ -87,10 +87,16 @@ export async function buildBaseImage(
// Trust the documentation ranges as plain-HTTP registries.
//
// A scenario's registry is scenery inside the scenario, serving over HTTP, and a runtime
// will not pull from one without being told. Scoped to RFC 5737 and RFC 3849 ranges rather
// than a specific address, because those never route on the real internet — so this cannot
// make a real machine trust a real registry, whatever it is copied onto.
// **Kept after the lab's own registry was deleted, because it was never only for that.** The
// mesh HAS a registry — the `registry` module, `mesh-registry`, serving artifacts to the whole
// mesh on port 5000 over plain HTTP from whatever node runs it. In a scenario that node's
// address is a documentation-range address, and a runtime will not pull from a plain-HTTP
// registry without being told to. Take this away and the artifact store is unusable from every
// machine but the one hosting it.
//
// Scoped to RFC 5737 and RFC 3849 ranges rather than a specific address, because those never
// route on the real internet — so this cannot make a real machine trust a real registry,
// whatever it is copied onto.
await incus([
"exec", BUILDER, "--", "sh", "-c",
`mkdir -p /etc/docker && printf '%s' '${JSON.stringify({
@@ -162,8 +168,8 @@ export async function buildBaseImage(
// Read back that the runtime will actually pull over plain HTTP from a documentation
// range. Writing the file is not the same as the daemon honouring it, and a base image
// that looks right here fails much later — in a sealed scenario, as a container that
// cannot fetch its image, which is a long way from the cause.
// that looks right here fails much later — as a container that cannot fetch its image
// from the mesh's own artifact store, which is a long way from the cause.
const trusted = await incusOk(
["exec", BUILDER, "--", "docker", "info", "--format", "{{.RegistryConfig.InsecureRegistryCIDRs}}"],
60_000,
@@ -172,7 +178,8 @@ export async function buildBaseImage(
throw new BaseImageError(
`the runtime in ${BUILDER} does not trust the documentation ranges as plain-HTTP ` +
`registries. It reported: ${trusted?.trim() || "nothing"}\n` +
` Every scenario raised from this image would fail to pull from its own registry.`,
` Every scenario raised from this image would fail to pull from the mesh's own ` +
`artifact store, which serves plain HTTP inside the scenario.`,
);
}
log(" trusts the documentation ranges as registries");
src/lifecycle/egress.ts
+126
View File
@@ -0,0 +1,126 @@
/**
* Confirming a machine that says it can reach the outside actually can.
*
* **This is what the registry-reachability check became.** The old one proved that every machine
* could fetch a manifest from the registry the lab raised inside the scenario — a real check of a
* fake path, since no production mesh has such a registry. What a machine actually does is pull
* from the internet, and that is now the thing worth proving before a raise says it is finished.
*
* The failure it exists to stop is the same one, in the same shape: `raise` returns, the caller
* applies a substrate, the first pull fails, no node enrols, and the instance is left a bare
* shell — with the cause several steps back and looking like a mesh fault rather than a lab one.
*
* Two things are checked, in this order, because they fail differently and the difference is the
* whole diagnosis:
*
* - **A name resolves.** Without this the machine has a route and no way to use it, and every
* pull dies inside the runtime saying it cannot look up a host.
* - **The path carries.** A request to the registry every image ultimately comes from, over the
* uplink, through whatever gateway sits in front of this machine. Any HTTP answer counts: what
* is in question is the path, not whether Docker Hub likes us.
*/
import type { Scenario } from "../declaration/types.ts";
import { incus, incusOk } from "../incus/client.ts";
export class EgressError extends Error {
constructor(message: string) {
super(message);
this.name = "EgressError";
}
}
/** The host every image is fetched through, in the end. Asked for, never pulled from, here. */
const UPSTREAM = "registry-1.docker.io";
/**
* Confirm every machine declaring `egress` can resolve and reach the outside.
*
* Run after the routes and the firewalls, because that is the path a pull will take: a home node's
* default is the uplink, its route to the rest of the scenario is through its gateway, and its own
* filtering is in place. Checking earlier would prove something no pull relies on.
*/
export async function confirmEgress(
scenario: Scenario,
machineNames: Map<string, string>,
log: (message: string) => void = () => {},
waitSeconds = 120,
): Promise<void> {
for (const [machine, spec] of Object.entries(scenario.machines)) {
if (!spec.egress || spec.at === "detached") continue;
const name = machineNames.get(machine);
if (!name) continue;
if (!(await resolves(name, waitSeconds))) {
// One repair, then a verdict. The uplink is the lab's own network and its DHCP server is
// also its resolver, so the machine has been told the answer and may simply have nowhere
// to write it — an image without systemd-resolved leaves `UseDNS=yes` inert.
await pointResolverAtTheUplink(name);
if (!(await resolves(name, 30))) {
throw new EgressError(
`${machine} declares egress and cannot resolve ${UPSTREAM}.\n` +
` It has a route out and no way to use it, so every image pulled from the internet ` +
`would fail inside the runtime as a lookup error.\n` +
` The uplink's DHCP server is also its resolver; this machine has not taken it.`,
);
}
}
const code = await reaches(name, waitSeconds);
if (!code) {
throw new EgressError(
`${machine} declares egress, resolves names, and cannot reach ${UPSTREAM}.\n` +
` This is the PATH: its default route, the uplink, or the host's own forwarding. ` +
`Every third-party image this machine needs is pulled from the internet, so anything ` +
`applied to it would stop at the first container.`,
);
}
log(` ${machine} reaches the internet over its uplink (${UPSTREAM} answered ${code})`);
}
}
async function resolves(name: string, waitSeconds: number): Promise<boolean> {
const deadline = Date.now() + waitSeconds * 1_000;
while (Date.now() < deadline) {
const said = await incusOk(
["exec", name, "--", "sh", "-c", `getent hosts ${UPSTREAM} >/dev/null && echo yes`], 30_000,
);
if (said?.trim() === "yes") return true;
await new Promise((r) => setTimeout(r, 3_000));
}
return false;
}
/**
* Any HTTP status at all, which is what "the path carries" means.
*
* Not 200: an unauthenticated `/v2/` is answered 401 by design, and a check demanding 200 would
* fail on a machine whose network is perfect.
*/
async function reaches(name: string, waitSeconds: number): Promise<string | null> {
const deadline = Date.now() + waitSeconds * 1_000;
while (Date.now() < deadline) {
const said = (await incusOk(
["exec", name, "--", "sh", "-c",
`curl -s -o /dev/null -w '%{http_code}' --max-time 15 https://${UPSTREAM}/v2/`], 40_000,
))?.trim();
if (said && /^[1-5][0-9]{2}$/.test(said)) return said;
await new Promise((r) => setTimeout(r, 5_000));
}
return null;
}
/**
* Write a resolver of last resort: the uplink's own gateway, which serves DHCP and DNS both.
*
* Deliberately the machine's default next hop rather than a name looked up somewhere — for a
* machine with egress that is the uplink by construction, since every scenario range is routed
* explicitly and nothing else defaults.
*/
async function pointResolverAtTheUplink(name: string): Promise<void> {
await incus([
"exec", name, "--", "sh", "-c",
`via=$(ip -4 route show default | awk '{print $3}' | head -n1); ` +
`[ -n "$via" ] && printf 'nameserver %s\\n' "$via" > /etc/resolv.conf; true`,
], 30_000);
}
src/lifecycle/operate.ts
+36 -2
View File
@@ -61,11 +61,45 @@ export async function exec(
*/
timeoutMs = 120_000,
): Promise<{ stdout: string; stderr: string }> {
const name = await instanceNameOf(instanceId, machine);
return incus(["exec", name, "--", ...command], timeoutMs);
}
/**
* What the hypervisor calls one of a scenario's machines.
*
* Asked of the daemon by the metadata each instance carries, and only derived from the naming
* rule when it answers nothing — the same order `exec` has always used. It is exported because
* the placement stage takes this name rather than the pair, and a caller that wants to put
* something on one machine after the raise (the installer, a catalogue checkout) would otherwise
* have to reimplement the lookup and get the fallback wrong.
*/
export async function instanceNameOf(instanceId: string, machine: string): Promise<string> {
const found = (await taggedInstances()).find(
(i) => i.instanceId === instanceId && i.machine === machine,
);
const name = found?.name ?? machineName(instanceId, machine);
return incus(["exec", name, "--", ...command], timeoutMs);
return found?.name ?? machineName(instanceId, machine);
}
/**
* Put a file from this workstation inside a machine.
*
* The long default timeout is not generosity: what goes through here is an installer carrying a
* container image, which is tens of megabytes, and a push that is merely slow must not look like
* a push that is stuck.
*/
export async function push(
instanceId: string,
machine: string,
local: string,
remote: string,
mode?: string,
timeoutMs = 900_000,
): Promise<void> {
const name = await instanceNameOf(instanceId, machine);
const args = ["file", "push", local, `${name}${remote}`];
if (mode) args.push("--mode", mode);
await incus(args, timeoutMs);
}
/**
src/lifecycle/place.ts
+210 -5
View File
@@ -19,6 +19,7 @@ import { join } from "node:path";
import { incus, incusOk, succeeds } from "../incus/client.ts";
import { around, log, shorten } from "../log.ts";
import { mustBeHandedOver, repositoryOf, type HeldImage } from "../pinning.ts";
/**
* What this stage can put inside a machine.
@@ -41,6 +42,17 @@ export function isPlaceable(artifact: string): boolean {
/** Where the host binary lives on a machine once placed. */
export const HOST_PATH = "/usr/local/bin/mesh-host";
/**
* Where the installer lives on a machine once placed.
*
* **Beside the host, because it is the same tier and the same delivery** (novox/hq ADR 0067):
* bootstrapping is done by hand and it changes a machine, which is what tier 0 is — but `mesh-host`
* says of itself that it connects to nothing and listens on nothing, and an installer that loads
* images and interrogates a control plane cannot be folded into it without making that sentence
* false. Two programs, one shelf.
*/
export const BOOTSTRAP_PATH = "/usr/local/bin/mesh-bootstrap";
export interface Placement {
machine: string;
artifacts: string[];
@@ -77,6 +89,14 @@ export function hostBinaryPath(): string | null {
return process.env["MESH_LAB_HOST_BINARY"] ?? null;
}
/**
* The installer to place, from the environment. Same rule as the host binary: an explicit path,
* and nothing is guessed.
*/
export function bootstrapBinaryPath(): string | null {
return process.env["MESH_LAB_BOOTSTRAP_BINARY"] ?? null;
}
export class PlacementError extends Error {
readonly machine: string;
@@ -145,6 +165,41 @@ export async function placeHost(
return { machine, profile, version };
}
/**
* Put the installer on a machine and ask it what it is.
*
* The same shape as {@link placeHost} and for the same reason: the version is read back from the
* running binary, because a file arriving is not a program working (novox/hq ADR 0018). The
* installer is the larger of the two by an order of magnitude — it carries a saved container image
* — so a copy that half-arrived is a real possibility rather than a theoretical one.
*
* It is placed on ONE machine, not all of them. Only the anchor is bootstrapped; every other
* machine joins a mesh that already exists, with the host binary and a token and nothing else.
*/
export async function placeBootstrap(
instanceName: string,
machine: string,
binary: string,
log: (message: string) => void = () => {},
): Promise<string> {
await incus(["file", "push", binary, `${instanceName}${BOOTSTRAP_PATH}`, "--mode", "0755"],
900_000);
const version = (await incusOk(
["exec", instanceName, "--", BOOTSTRAP_PATH, "version"], 60_000,
))?.trim();
if (!version) {
throw new PlacementError(
machine,
`the installer was copied to ${machine} and does not run there. It carries a saved ` +
`container image and is twenty megabytes or so, which is exactly the size at which a ` +
`truncated copy stops being theoretical.`,
);
}
log(` placed the installer on ${machine} (${version})`);
return version;
}
/** Place everything a scenario declares. */
export async function applyPlacements(
scenario: Scenario,
@@ -263,17 +318,17 @@ export async function placeImage(
// sealed machine cannot reach. Measured, not assumed: the load says `Loaded image ID:`
// instead of `Loaded image:`, and `docker images` then lists nothing.
//
// This collides with novox/hq ADR 0006, which pins bundle images BY DIGEST and has the host
// refuse anything else. Reconciling the two needs a registry inside the scenario, which is
// real design work — see 04-ISSUES/009.
// What an archive DOES keep is the image's own ID — the digest of its configuration — and that
// is how the mesh's own images are named once loaded. See {@link loadHeldImages}.
if (reference.includes("@sha256:")) {
throw new PlacementError(
machine,
`${reference} is pinned by digest, and an image placed from an archive cannot keep its ` +
`digest — a repo digest only exists for an image a registry served.\n` +
` Placing it would load an image with no name, and a container declaring that digest ` +
`would try to reach a registry the machine cannot see.\n` +
` Place it by tag, or give the scenario a registry (novox/hq 04-ISSUES/009).`,
`would try to reach a registry.\n` +
` Place it by tag. What survives being loaded is the image's own ID, which is what a ` +
`declaration names it by (mesh-host: an image the machine already holds).`,
);
}
@@ -397,3 +452,153 @@ async function waitForRuntime(instanceName: string, machine: string): Promise<vo
` systemd is waiting on:\n${jobs.trim() || " (it said nothing)"}`,
);
}
// --- the mesh's own images ----------------------------------------------------------------------
/**
* Which machine is handed which of the mesh's own images.
*
* A machine that says nothing gets the lot, which is right for a one-machine bed. A machine that
* lists some gets those. This is where a workstation running one small module stops paying for
* forty runtimes it will never start.
*/
export function planHeldImages(scenario: Scenario): { machine: string; images: string[] }[] {
const all = scenario.images ?? [];
if (all.length === 0) return [];
return Object.entries(scenario.machines)
.map(([machine, spec]) => ({ machine, images: spec.images ?? all }))
.filter((plan) => plan.images.length > 0);
}
/**
* Put the mesh's own images onto the machines that need them, and say what they are now called.
*
* **This is what replaced the lab's registry**, and the difference is the whole point. A registry
* inside the scenario served every image — third-party ones included — from an address that exists
* in no production mesh, so a bootstrap that could only work against it went green here and would
* have failed anywhere else. There is no such registry now: third-party images are pulled from the
* internet over each machine's `egress` uplink, and the mesh's own arrive the way they arrive on an
* operator's machine — somebody built them and put them there.
*
* The reference a declaration then uses is the image's **own ID**, the digest of its configuration.
* `docker load` preserves it, so the name is identical on the workstation that built the image and
* on every machine handed a copy — immutable, unforgeable, and requiring nothing to have served it
* (mesh-host, *an image may be named by the digest of its own configuration*).
*
* Read back on both sides. The ID is taken from the workstation and then CONFIRMED on the machine,
* because a load that lands a different image than the one exported is exactly the silent fault
* this lab exists to catch — and the reference is what every manifest will be rewritten to.
*/
export async function loadHeldImages(
scenario: Scenario,
machineNames: Map<string, string>,
log: (message: string) => void = () => {},
): Promise<HeldImage[]> {
const plans = planHeldImages(scenario);
if (plans.length === 0) return [];
const held: HeldImage[] = [];
for (const requested of scenario.images ?? []) {
// Refused by the validator, so reaching here would be a validator bug — but the consequence
// is a third-party image quietly loaded from the workstation instead of pulled, which is the
// fiction all of this exists to remove. Cheap to check, expensive to miss.
if (!mustBeHandedOver(requested)) {
throw new Error(
`images: '${requested}' is not one of the mesh's own images. It is pulled from the ` +
`internet by the machine that needs it, not loaded from this workstation.`,
);
}
const wanted = plans.filter((plan) => plan.images.includes(requested)).map((p) => p.machine);
if (wanted.length === 0) continue;
const onThisWorkstation = (await local(
"docker", ["image", "inspect", "--format", "{{.Id}}", requested], 60_000,
)).stdout.trim();
if (!/^sha256:[0-9a-f]{64}$/.test(onThisWorkstation)) {
throw new Error(
`${requested} is not on this workstation, so there is nothing to hand the machines.\n` +
` It is one of the mesh's own images and exists in no registry — nothing can pull it.\n` +
` Build it first (mesh-control's \`make image …\`, or scripts/build-module-runtime.sh).`,
);
}
// **An image id belongs to the runtime holding it, and does not survive the journey.** It is
// the digest of the image's *configuration*, and a runtime rewrites that configuration as it
// loads: this workstation saves in one format and the machine's older runtime stores it in
// another, so the same bytes arrive under a different name. Measured, not assumed — the same
// image was b86bb81c… here and 2dc21904… on the machine.
//
// So the id is READ BACK from the machine rather than predicted from here. Predicting it is
// what the earlier version did, and it failed at the only useful moment: the manifests would
// have been rewritten to a reference no machine holds, and nothing serves these images, so
// every apply would have stopped at the container with a pull that cannot succeed.
let reference = "";
// Exported once, handed to each machine that asked for it. The archive is the expensive part
// and it does not depend on the destination.
const tar = join(tmpdir(), `mesh-lab-held-${process.pid}-${Date.now()}.tar`);
const saved = await local("docker", ["save", requested, "-o", tar], 900_000);
if (!saved.ok) {
await unlink(tar).catch(() => {});
throw new Error(`cannot export ${requested} from this workstation: ${saved.stderr.trim()}`);
}
try {
for (const machine of wanted) {
const name = machineNames.get(machine);
if (!name) continue;
await waitForRuntime(name, machine);
await incus(["file", "push", tar, `${name}/tmp/held.tar`], 900_000);
const loaded = await incusOk(
["exec", name, "--", "docker", "load", "-i", "/tmp/held.tar"], 900_000,
);
if (!loaded?.includes("Loaded image")) {
throw new PlacementError(
machine,
`${requested} was pushed to ${machine} and did not load.\n` +
` The runtime said: ${loaded?.trim() || "nothing"}`,
);
}
// What this machine calls it, asked of the tag it was just loaded under. This is the
// reference every manifest naming this image will be rewritten to.
const there = (await incusOk(
["exec", name, "--", "docker", "image", "inspect", "--format", "{{.Id}}", requested],
120_000,
))?.trim() ?? "";
if (!/^sha256:[0-9a-f]{64}$/.test(there)) {
throw new PlacementError(
machine,
`${requested} loaded onto ${machine} and the runtime will not say what it holds.\n` +
` It answered '${there || "nothing"}'.\n` +
` Nothing serves this image, so a manifest naming it has only what the machine ` +
`itself reports — and there is nothing to fall back to.`,
);
}
// **A manifest carries one reference, so the machines must agree on it.** They are built
// from one base image and load one archive, so they do; if that ever stops being true the
// image cannot be named at all from a catalogue, and that is worth stopping for rather
// than rewriting to whichever machine answered last.
if (!reference) {
reference = there;
} else if (reference !== there) {
throw new PlacementError(
machine,
`${requested} is ${there} on ${machine} and ${reference} on a machine already ` +
`loaded.\n` +
` One manifest cannot name both, and this image exists in no registry to be ` +
`named by instead. The machines' runtimes differ in a way that changes how they ` +
`store what they are given.`,
);
}
await succeeds(["exec", name, "--", "rm", "-f", "/tmp/held.tar"], 60_000);
}
} finally {
await unlink(tar).catch(() => {});
}
held.push({ requested, repository: repositoryOf(requested), reference });
log(` ${requested} → ${reference.slice(0, 19)}… on ${wanted.join(", ")}`);
}
return held;
}
src/lifecycle/raise.ts
+25 -26
View File
@@ -22,9 +22,10 @@ import { applyAddresses, applyDefaultRoutes } from "./address.ts";
import { assertSupported } from "./supported.ts";
import { planRouters, raiseRouters, raiseTransit } from "./router.ts";
import { applyHostFirewalls } from "./firewall.ts";
import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements } from "./place.ts";
import { IMAGE_PREFIX, BASE_IMAGE_ALIAS, BASE_IMAGE_HOWTO, planPlacements, applyPlacements, loadHeldImages } from "./place.ts";
import { baseImageExists, UPSTREAM_IMAGE } from "./base.ts";
import { discardStock, raiseRegistry, stockRegistry } from "./registry.ts";
import { confirmEgress } from "./egress.ts";
import type { HeldImage } from "../pinning.ts";
import { log as record } from "../log.ts";
/** Drivers whose snapshots are copy-on-write. On `dir` a snapshot is a full copy. */
@@ -47,12 +48,15 @@ export interface RaisedScenario {
networks: string[];
pool: string;
/**
* Images the scenario's registry serves, as references a declaration can pin.
* The mesh's own images, as loaded onto the machines, and what a declaration should call them.
*
* Reported rather than declared, because the digest is the one this registry assigned and
* is not knowable before it was raised.
* Reported rather than declared: an image built from source has no digest until it has been
* built, and what names it here is the digest of its own configuration.
*
* **Only ours.** Everything third-party is pulled from the internet by the machine that needs
* it, so it is not in this list and nothing rewrites it.
*/
images: string[];
images: HeldImage[];
}
export class RaiseError extends Error {
@@ -314,24 +318,6 @@ export async function raise(
const routers = await raiseRouters(scenario, instanceId, planRouters(scenario, instanceId), log);
if (transit) routers.push(transit);
// Stocked on this workstation, where there is a network, and served from inside the
// scenario, where there is not (novox/hq 04-ISSUES/009).
enter("stocking the registry");
const stock = await stockRegistry(scenario.images ?? [], log);
let registry: Awaited<ReturnType<typeof raiseRegistry>> = null;
try {
enter("raising the registry");
registry = await raiseRegistry(scenario, instanceId, stock, log);
} finally {
// Cleaning up scratch must not fail a raise that succeeded. The scenario is standing
// and usable; a directory left behind is untidy, and saying so is the honest report.
try {
await discardStock(stock);
} catch (err) {
log(` (could not remove the registry's scratch directory: ${(err as Error).message})`);
}
}
enter("routing machines through their gateways");
await applyDefaultRoutes(scenario, byMachine, log);
@@ -340,16 +326,29 @@ export async function raise(
enter("applying host firewalls");
await applyHostFirewalls(scenario, byMachine, log);
// **Only now is "this machine can reach the outside" a true statement.** The route, the
// gateway and the machine's own filtering are all in place, so this is the path a pull takes.
// A raise that returned without checking would hand the next step a fact it depends on and
// has no way to test — which is how a substrate apply used to die on its first pull.
enter("confirming egress reaches the internet");
await confirmEgress(scenario, byMachine, log);
// Last, and only once the underlay is real. Placing before the machines can reach each
// other would test the host against a network the scenario does not describe.
enter("placing");
await applyPlacements(scenario, byMachine, log);
// After `placing`, because loading an image needs the container runtime that `placing`
// confirmed. The mesh's own images only — everything third-party is pulled by the machine
// itself, over its uplink, exactly as it is on a real one.
enter("loading the mesh's own images onto the machines");
const images = await loadHeldImages(scenario, byMachine, log);
return {
instanceId,
scenario: scenario.scenario,
images: registry?.pinned ?? [],
machines: [...created, ...routers, ...(registry ? [registry.machine] : [])],
images,
machines: [...created, ...routers],
networks,
pool,
};
src/lifecycle/registry.ts
-405
View File
@@ -1,405 +0,0 @@
/**
* A registry inside the scenario.
*
* A sealed machine cannot reach a registry, and an image placed from an archive cannot keep its
* digest — `docker save` of a digest reference produces an archive with no repo tag, because a
* repo digest only exists for an image a registry served (novox/hq 04-ISSUES/009). So an image
* pinned by digest, which is the only kind the host accepts
* ([ADR 0006](../../02-DECISIONS/0046-the-installer-fetches-what-it-pins.md)), could not be
* placed at all.
*
* The answer is a registry, and it is not a workaround for the lab: ADR 0006 names an OCI
* registry as substrate, and ADR 0006 says a first node fetches "upstream, wherever the image
* ordinarily lives". **This is that upstream** — scenery, like the transit router is the
* internet ([ADR 0016](../../02-DECISIONS/0033-a-router-is-scenery-not-a-node.md)).
*
* The digests it serves are its own, not Docker Hub's, and that is correct rather than a
* compromise. What ADR 0006 requires is a reference that is exact and cannot move. A digest
* assigned by this registry is both.
*/
import { spawn } from "node:child_process";
import { incus, incusOk, succeeds } from "../incus/client.ts";
import { macFor, networkName } from "./names.ts";
import { addressLink } from "./address.ts";
import { around, log, shorten } from "../log.ts";
import { BASE_IMAGE_ALIAS, placeImage } from "./place.ts";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
/** The image the registry itself runs from. Placed by tag, which archives keep. */
export const REGISTRY_IMAGE = "registry:2";
/** Where the registry serves, inside its machine. */
export const REGISTRY_PORT = 5000;
export class RegistryError extends Error {
constructor(message: string) {
super(message);
this.name = "RegistryError";
}
}
export interface StockedImage {
/** What the scenario asked for, as written. */
requested: string;
/** The repository path the registry serves it under. */
repository: string;
/** The digest THIS registry assigned. What a declaration pins. */
digest: string;
}
export interface Stock {
/** A directory holding the registry's data, ready to be placed in a machine. */
dataDir: string;
images: StockedImage[];
}
/**
* Build a registry's data directory on this workstation, with the given images in it.
*
* Runs a throwaway registry here — where there IS a network — pushes into it, and keeps what
* it wrote. Research 012's reframing again: fetch at build time on a machine that has a
* network, apply on a target that needs nothing.
*
* The caller owns the returned directory and must remove it.
*/
export async function stockRegistry(
references: string[],
log: (message: string) => void = () => {},
): Promise<Stock> {
if (references.length === 0) return { dataDir: "", images: [] };
const dataDir = await mkdtemp(join(tmpdir(), "mesh-lab-registry-"));
const container = `mesh-lab-stock-${process.pid}`;
const port = 5000 + (process.pid % 1000);
await docker(["rm", "-f", container], 60_000);
const started = await docker(
["run", "-d", "--name", container, "-p", `${port}:5000`, "-v", `${dataDir}:/var/lib/registry`,
REGISTRY_IMAGE],
300_000,
);
if (!started.ok) {
await rm(dataDir, { recursive: true, force: true });
throw new RegistryError(
`cannot run ${REGISTRY_IMAGE} on this workstation to stock a registry: ${started.stderr.trim()}`,
);
}
try {
await waitForRegistry(port);
const images: StockedImage[] = [];
for (const reference of references) {
// The repository path a machine will pull from. A tag is dropped: what a declaration
// pins is the digest, and carrying the tag as well would invite pinning the wrong one.
const repository = repositoryFor(reference);
const target = `localhost:${port}/${repository}`;
const tagged = await docker(["tag", reference, target], 60_000);
if (!tagged.ok) {
throw new RegistryError(
`${reference} is not on this workstation, and the lab does not fetch on a scenario's ` +
`behalf. Pull it here first.\n ${tagged.stderr.trim()}`,
);
}
const pushed = await docker(["push", target], 900_000);
if (!pushed.ok) throw new RegistryError(`cannot push ${reference}: ${pushed.stderr.trim()}`);
const digest = digestFrom(pushed.stdout + pushed.stderr);
if (!digest) {
throw new RegistryError(
`${reference} was pushed and the registry did not report a digest. Without one there ` +
`is nothing for a declaration to pin.`,
);
}
images.push({ requested: reference, repository, digest });
log(` stocked ${repository}@${digest}`);
}
return { dataDir, images };
} catch (err) {
await discardStock({ dataDir, images: [] });
throw err;
} finally {
await docker(["rm", "-f", container], 60_000);
}
}
/**
* Remove a stocked registry's data.
*
* Through a container, because a container wrote it. The registry runs as root inside, so the
* blobs it writes into a bind mount are owned by root and an ordinary process cannot remove
* them — `rmdir` fails with EACCES on a directory that looks like ours.
*
* Whoever made the files removes them.
*/
export async function discardStock(stock: Stock): Promise<void> {
if (!stock.dataDir) return;
await docker(["run", "--rm", "-v", `${stock.dataDir}:/stock`, REGISTRY_IMAGE,
"sh", "-c", "rm -rf /stock/* /stock/.[!.]* 2>/dev/null || true"], 120_000);
await rm(stock.dataDir, { recursive: true, force: true }).catch(() => {});
}
/** `alpine:3.20` and `alpine` both serve from `alpine`; `foo/bar:1` from `foo/bar`. */
export function repositoryFor(reference: string): string {
const withoutDigest = reference.split("@")[0] ?? reference;
const lastColon = withoutDigest.lastIndexOf(":");
const lastSlash = withoutDigest.lastIndexOf("/");
return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
}
/** `docker push` prints `<tag>: digest: sha256:… size: …` on its last useful line. */
export function digestFrom(output: string): string | null {
const match = output.match(/digest:\s*(sha256:[a-f0-9]{64})/);
return match?.[1] ?? null;
}
async function waitForRegistry(port: number): Promise<void> {
for (let i = 0; i < 30; i++) {
const probe = await docker(["run", "--rm", "--network", "host", REGISTRY_IMAGE,
"sh", "-c", `wget -q -O- http://localhost:${port}/v2/ >/dev/null 2>&1`], 30_000);
if (probe.ok) return;
await new Promise((r) => setTimeout(r, 1_000));
}
throw new RegistryError("a registry was started on this workstation and never answered");
}
function docker(
args: string[],
timeoutMs: number,
): Promise<{ ok: boolean; stdout: string; stderr: string }> {
// The second of the three places the lab runs an external program (novox/hq 04-ISSUES/024).
// `docker push` of a large image is minutes of legitimate silence, which is exactly when a
// heartbeat earns its keep.
return around(`docker ${shorten(args)}`, () => runDocker(args, timeoutMs), { heartbeatMs: 15_000 });
}
function runDocker(
args: string[],
timeoutMs: number,
): Promise<{ ok: boolean; stdout: string; stderr: string }> {
return new Promise((resolve) => {
const child = spawn("docker", args, { stdio: ["ignore", "pipe", "pipe"] });
let stdout = "";
let stderr = "";
const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
child.stdout.on("data", (d) => (stdout += d));
child.stderr.on("data", (d) => (stderr += d));
child.on("error", (err) => {
clearTimeout(timer);
resolve({ ok: false, stdout, stderr: err.message });
});
child.on("close", (code) => {
clearTimeout(timer);
// A docker failure is an answer here rather than an exception, so it would otherwise pass
// through the log looking exactly like a success.
if (code !== 0) {
log.debug(` exit ${code}: ${shorten([stderr.trim() || "(nothing on stderr)"], 400)}`);
}
resolve({ ok: code === 0, stdout, stderr });
});
});
}
// --- the registry inside a scenario ------------------------------------------------------------
/**
* Where the registry sits on its segment.
*
* A convention rather than a declaration, like the router's. `.250` is chosen to sit well away
* from the low addresses scenarios give their machines, so a scenario can be written without
* thinking about it and a collision is obvious when it happens.
*/
export const REGISTRY_HOST_OCTET = 250;
/** The address the registry answers on, given the segment it is attached to. */
export function registryAddress(cidr: string): string {
const [network] = cidr.split("/");
const parts = (network ?? "").split(".");
if (parts.length !== 4) {
throw new RegistryError(
`cannot place a registry on '${cidr}': it is not an IPv4 network, and the registry needs ` +
`an address a machine can be pointed at.`,
);
}
return `${parts[0]}.${parts[1]}.${parts[2]}.${REGISTRY_HOST_OCTET}`;
}
/** What a declaration should pin, once a scenario is raised. */
export function pinnedReference(address: string, image: StockedImage): string {
return `${address}:${REGISTRY_PORT}/${image.repository}@${image.digest}`;
}
// --- raising it inside a scenario ---------------------------------------------------------------
/** What a raised registry is, and what a declaration needs from it. */
export interface RaisedRegistry {
machine: string;
segment: string;
address: string;
/** Each image, as a reference a declaration can pin. */
pinned: string[];
}
/**
* Pick the segment the registry sits on.
*
* A public segment, because that is what stands in for the outside world — a first node fetches
* from upstream, and this is upstream. An IPv4 range, because a machine has to be pointed at it
* by address.
*/
export function registrySegment(
segments: Record<string, { kind: string; cidr: string[] }>,
): { name: string; cidr: string } | null {
for (const [name, segment] of Object.entries(segments)) {
if (segment.kind !== "public") continue;
const v4 = segment.cidr.find((c) => !c.includes(":"));
if (v4) return { name, cidr: v4 };
}
return null;
}
/**
* Raise a registry inside the scenario and load the stocked images into it.
*
* Scenery, in the same sense the transit router is: nothing under test runs on it, it holds no
* identity, and no assertion is made about its internals. It exists so that a machine can fetch
* an image the way a real one does — over the network, from a registry, by digest.
*/
export async function raiseRegistry(
scenario: { segments: Record<string, { kind: string; cidr: string[] }> },
instanceId: string,
stock: Stock,
log: (message: string) => void = () => {},
): Promise<RaisedRegistry | null> {
if (stock.images.length === 0) return null;
const segment = registrySegment(scenario.segments);
if (!segment) {
throw new RegistryError(
`this scenario declares images and has no public IPv4 segment to serve them from.\n` +
` The registry stands in for the outside world, so it sits on a public segment.`,
);
}
const address = registryAddress(segment.cidr);
const name = `mlab-${instanceId}-registry`;
const prefix = segment.cidr.slice(segment.cidr.lastIndexOf("/"));
if (!(await succeeds(["config", "show", name], 15_000))) {
await incus([
"init", BASE_IMAGE_ALIAS, name, "--vm",
"-c", "security.secureboot=false",
"-c", "limits.memory=1GiB",
"-c", `user.mesh-lab.instance=${instanceId}`,
// Tagged as a machine as well, so `destroy` finds it with one query — a router that
// carried only its own tag was left behind and held its networks open.
"-c", "user.mesh-lab.machine=registry",
"-c", "user.mesh-lab.registry=true",
], 300_000);
await succeeds(["config", "device", "remove", name, "eth0"], 15_000);
await incus([
"config", "device", "add", name, "eth0", "nic",
"nictype=bridged",
`parent=${networkName(instanceId, segment.name)}`,
`hwaddr=${macFor(instanceId, "registry", 0)}`,
]);
}
await succeeds(["start", name], 60_000);
await waitForAgent(name);
// Addressed the way every other machine is: a systemd-networkd unit matching the MAC.
//
// **This used to be `ip addr add`, and it stalled the lab.** An address set by hand leaves
// networkd waiting to configure a link it was never told about, so the link sits at
// `configuring`, `systemd-networkd-wait-online` never returns — its timeout is `infinity` —
// and `network-online.target` is never reached. Docker is ordered after that target, so
// `docker load` two lines below blocked on a socket whose daemon was queued behind a target
// that would never come.
//
// Matching on MAC and not on interface name is still the rule: a machine with a container
// runtime has a `docker0` that sorts before `enp5s0`, and naive selection configures that.
await addressLink(name, {
device: "eth0",
mac: macFor(instanceId, "registry", 0),
addresses: [`${address}${prefix}`],
// The registry takes the segment's default. It carried no MTU before this and still does
// not: what a scenario sets an MTU for is the path under test, and this is scenery.
mtu: undefined,
});
log(` registry on ${segment.name} at ${address}`);
// The registry's own image, placed by tag — an archive keeps a tag and cannot keep a digest,
// which is the whole reason this machine exists.
// Logged, not silenced. This is the step a stall sat in for thirty-five minutes while the
// caller had passed it a callback that threw everything away (novox/hq 04-ISSUES/024).
await placeImage(name, "registry", REGISTRY_IMAGE, log);
// The destination must EXIST before a recursive push, or incus copies the source's contents
// rather than the source — the data lands one directory too shallow, the registry finds
// nothing where it looks, and every pull fails with `not found`.
await incus(["exec", name, "--", "mkdir", "-p", "/srv/registry"], 60_000);
await incus(["file", "push", "-r", `${stock.dataDir}/docker`, `${name}/srv/registry/`], 900_000);
await incus(["exec", name, "--", "docker", "run", "-d",
"--name", "registry", "--restart", "unless-stopped",
"-p", `${REGISTRY_PORT}:5000`,
"-v", "/srv/registry:/var/lib/registry",
REGISTRY_IMAGE], 300_000);
// Read back that each image is SERVED, by asking for its manifest by digest — which is
// exactly what a machine will do.
//
// Not that the catalog endpoint answers: `{"repositories":[]}` contains the word
// `repositories`, so checking for that passed on a registry holding nothing at all, and the
// failure surfaced much later as a container that could not be pulled.
let answered = false;
for (let i = 0; i < 20 && !answered; i++) {
const ping = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null",
"-w", "%{http_code}", "--max-time", "3",
`http://localhost:${REGISTRY_PORT}/v2/`], 30_000);
answered = ping?.trim() === "200";
if (!answered) await new Promise((r) => setTimeout(r, 2_000));
}
if (!answered) {
throw new RegistryError(
`the registry on ${name} started and never answered. Machines in this scenario cannot ` +
`fetch an image, so nothing that declares a container will work.`,
);
}
const pinned: string[] = [];
for (const image of stock.images) {
const code = await incusOk(["exec", name, "--", "curl", "-s", "-o", "/dev/null",
"-w", "%{http_code}", "--max-time", "5",
"-H", "Accept: application/vnd.docker.distribution.manifest.v2+json",
`http://localhost:${REGISTRY_PORT}/v2/${image.repository}/manifests/${image.digest}`,
], 60_000);
if (code?.trim() !== "200") {
throw new RegistryError(
`the registry on ${name} is running and does not serve ${image.repository}@${image.digest} ` +
`(it answered ${code?.trim() || "nothing"}).\n` +
` The images were stocked on this workstation and did not arrive intact, so a ` +
`machine declaring that image would fail to pull it.`,
);
}
const reference = pinnedReference(address, image);
pinned.push(reference);
log(` serving ${reference}`);
}
return { machine: name, segment: segment.name, address, pinned };
}
async function waitForAgent(name: string): Promise<void> {
for (let i = 0; i < 90; i++) {
if (await succeeds(["exec", name, "--", "true"], 10_000)) return;
await new Promise((r) => setTimeout(r, 2_000));
}
throw new RegistryError(`${name} started and its agent never answered.`);
}
src/pinning.ts
+114 -28
View File
@@ -1,55 +1,141 @@
/**
* Rewriting an image reference to the one a scenario's own registry serves.
* Naming the mesh's own images by what they are.
*
* **A digest is not knowable until something is built** (novox/hq 04-ISSUES/025). A manifest in a
* repository can pin a third-party image, because somebody can ask a registry what a tag points
* at. It cannot pin an image the mesh builds itself: that image does not exist yet, and when it
* does its digest belongs to whichever registry served it.
* at. It cannot pin an image the mesh builds itself: mesh-control, mesh-builder, mesh-route-proxy,
* the per-module runtimes and the provisioners exist in no registry, so there is no manifest
* digest to write down. The catalogue ships sixty-four zeros for them, which parses, resolves,
* composes — and stops on the machine.
*
* The bundle has always had this problem and solves it by rewriting references once the scenario's
* registry is up and its digests are known. Modules have exactly the same problem and were solving
* it by shipping sixty-four zeros, which parses, resolves, composes — and stops on the machine.
* The lab used to answer that with a registry of its own: raise one inside the scenario, push
* everything into it, and rewrite every reference — third-party ones included — to the digest it
* assigned. **That registry does not exist in production, so the lab was testing a fiction**, and
* the fiction hid the bootstrap problems it was supposed to find.
*
* So the rewriting is shared rather than copied, and matches on the **repository**, because that
* is the part a person writes and the only part that survives being served somewhere else.
* What is true instead is two things:
*
* - **Third-party images are pulled from the internet.** They are left exactly as written, and
* the machine fetches them over its `egress` uplink the way any machine does.
* - **The mesh's own images are built and handed over.** They are loaded onto the machine from
* the workstation that built them, and named by the digest of their own image configuration —
* a bare `sha256:…`, which mesh-host accepts as "an image this machine already holds"
* (mesh-host, *an image may be named by the digest of its own configuration*).
*
* So the rewriting that remains is only the second kind, and it matches on the **repository**,
* because that is the part a person writes and the only part a placeholder digest does not say.
*/
/** `192.0.2.250:5000/ghcr.io/mailu/admin@sha256:…` → `ghcr.io/mailu/admin` */
export function repositoryOf(pinned: string): string {
const at = pinned.indexOf("@");
const body = at === -1 ? pinned : pinned.slice(0, at);
const slash = body.indexOf("/");
// Everything after the registry. A reference with no slash at all is its own repository.
return slash === -1 ? body : body.slice(slash + 1);
/** What the lab loaded onto a machine, and what a declaration should call it. */
export interface HeldImage {
/** As the scenario asked for it, a tag this workstation holds: `mesh-runtime-postgres:development`. */
requested: string;
/** What a manifest names it by, with no tag and no digest: `mesh-runtime-postgres`. */
repository: string;
/**
* The reference a declaration uses: a bare `sha256:<64 hex>`.
*
* The image's own ID — the digest of its configuration — which `docker load` preserves, so the
* name is the same on the workstation that built it and on every machine it was handed to.
*/
reference: string;
}
/** `alpine:3.20` and `alpine` both mean `alpine`; `foo/bar:1` means `foo/bar`. */
export function repositoryOf(reference: string): string {
const withoutDigest = reference.split("@")[0] ?? reference;
const lastColon = withoutDigest.lastIndexOf(":");
const lastSlash = withoutDigest.lastIndexOf("/");
return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
}
/**
* Replace every reference to a stocked repository with the reference this scenario serves.
* Whether a reference names an image the mesh builds for itself.
*
* Matching is on the repository and ignores whatever registry and digest were written down —
* a file may name `postgres@sha256:7456…` or `mesh-provision-postgres@sha256:0000…` and both mean
* *the postgres this scenario has*. That is the whole point: the text says which image, the
* scenario says which copy.
* **Derived from the shape the build produces, not from a list of names.** `make image
* builder-image provisioner-image objectstore-image redis-provisioner-image proxy-image` in
* mesh-control and `scripts/build-module-runtime.sh` here both tag their output `mesh-<something>`
* with no registry host and no upstream organisation — that is what "built here, published
* nowhere" looks like, and a hardcoded list would go stale the first time a module is added.
*
* A repository the scenario did not stock is left alone rather than blanked. It may be reachable
* some other way, and silently emptying a reference would produce the exact failure this exists to
* prevent.
* The absence of a slash carries the weight: `ghcr.io/mailu/admin` and
* `registry.example/novox/www` both say where they are fetched from, and a bare
* `mesh-runtime-plex` says there is nowhere.
*/
export function pinnedInto(text: string, served: string[]): string {
export function isMeshBuilt(reference: string): boolean {
const repository = repositoryOf(reference);
return !repository.includes("/") && repository.startsWith("mesh-");
}
/**
* Registries an anonymous pull works against.
*
* Not a list of what is trusted — a list of where no account is needed. Everything else wants one,
* and a scenario machine has none.
*/
const PUBLIC_REGISTRIES = [
"docker.io", "ghcr.io", "quay.io", "lscr.io", "gcr.io", "registry.k8s.io",
"public.ecr.aws", "mcr.microsoft.com", "docker.elastic.co", "registry.gitlab.com",
];
/** The registry a reference names, or "" when it names none and so means Docker Hub. */
export function registryOf(reference: string): string {
const first = repositoryOf(reference).split("/")[0] ?? "";
// A first segment is a registry only if it looks like a host: `novox/www` is an organisation on
// Docker Hub; `registry.example/novox/www` is somewhere else entirely.
return first.includes(".") || first.includes(":") ? first : "";
}
/**
* Whether the workstation has to hand this image over rather than let the machine fetch it.
*
* **Two reasons, one consequence.** An image the mesh builds for itself exists in no registry at
* all. An image in the operator's *private* registry exists in one the machines have no account
* for, and the pull fails with `no basic auth credentials` — which is not something more patience
* fixes. Either way the machine cannot get it alone, so the workstation, which does hold the
* credential, exports it and loads it.
*
* **This stands in for something, and it is worth saying what.** In a finished mesh these are built
* by the mesh's builder and published to the mesh's own store, and every machine pulls them from
* there with a credential the mesh granted it. Until that store exists there is nowhere for them to
* come from — and handing them over is the closest honest thing to it, rather than a registry the
* lab invents, which is exactly what was just removed.
*/
export function mustBeHandedOver(reference: string): boolean {
if (isMeshBuilt(reference)) return true;
const registry = registryOf(reference);
return registry !== "" && !PUBLIC_REGISTRIES.includes(registry);
}
/**
* Replace every reference to one of the mesh's own images with the image the machine holds.
*
* Matching is on the repository and ignores whatever digest was written down — a manifest says
* `mesh-runtime-postgres@sha256:0000…` and means *the runtime this machine was given*.
*
* **Everything else is left exactly as it is.** `postgres@sha256:7456…`, `gitea/gitea@sha256:…`
* and `ghcr.io/mailu/admin@sha256:…` are pulled from the internet over the machine's uplink, which
* is what a real machine does and the reason the lab's own registry is gone.
*/
export function pinnedInto(text: string, held: HeldImage[]): string {
let out = text;
for (const pinned of served) {
const repository = repositoryOf(pinned);
const escaped = repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
for (const image of held) {
const escaped = image.repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
// Optionally a registry, then the repository, then any digest. Anchored on a quote or
// whitespace so a longer repository ending in a shorter one is not half-replaced.
out = out.replaceAll(
new RegExp(`(?<=^|["\\s])(?:[A-Za-z0-9_.:-]+\\/)*${escaped}@sha256:[0-9a-f]{64}`, "g"),
pinned,
image.reference,
);
}
return out;
}
/** The reference for one repository, or undefined if this scenario loaded no such image. */
export function referenceFor(held: HeldImage[], repository: string): string | undefined {
return held.find((image) => image.repository === repository)?.reference;
}
/** Whether anything is still pinned to a placeholder, which would fail on the machine. */
export function stillUnpinned(text: string): string[] {
return [...text.matchAll(/([A-Za-z0-9_.:/-]+)@sha256:0{64}/g)].map((m) => m[1]!);
src/rebuild.ts
+33 -1
View File
@@ -2,7 +2,8 @@
* Rebuild what the lab runs, from source, before it runs.
*
* **A stale artifact reporting success against old rules is the fault this project keeps writing
* down** (novox/hq 04-ISSUES/005). The lab consumes three artifacts from two repositories, and they
* down** (novox/hq 04-ISSUES/005). The lab consumes a handful of artifacts from two repositories,
* and they
* were rebuilt by hand, one at a time, from memory. A rename in the control plane's catalogue needs
* both the control-plane image *and* the builder binary, because both parse manifests; rebuilding
* one left a binary eleven hours old refusing a field the mesh had just renamed, and cost a full
@@ -73,9 +74,40 @@ export function planned(env: NodeJS.ProcessEnv = process.env): Build[] {
});
}
}
// **The installer, carrying the control plane's image.**
//
// Last, and that is an ordering rather than a preference: `make bootstrap` embeds the output of
// `docker save <image>`, so the image has to have been built by the step above or the installer
// carries whatever was lying around — the eleven-hour-old artifact again, this time inside a
// binary where nothing would ever notice.
//
// It is built here at all because the bed now bootstraps THROUGH it (novox/hq ADR 0067): the
// anchor is brought into existence by running the same program a bare machine runs, rather than
// by the bed applying a substrate bundle by hand and calling that an install. An installer that
// was stale would be a bed proving something about last week's procedure.
const installer = env["MESH_LAB_BOOTSTRAP_BINARY"];
if (installer && where["mesh-host"]) {
builds.push({
what: "installer",
in: where["mesh-host"],
argv: ["make", "bootstrap", `IMAGE=${controlPlaneImage(env)}`, `BOOTSTRAP_OUT=${installer}`],
});
}
return builds;
}
/**
* The control-plane image the installer carries.
*
* `mesh-control:development` is what mesh-control's `make image` tags, and what the scenarios name
* — one tag, said in one place. It is overridable because a release installer carries a release
* image, and nothing about that is the lab's business.
*/
export function controlPlaneImage(env: NodeJS.ProcessEnv = process.env): string {
return env["MESH_LAB_CONTROL_IMAGE"] ?? "mesh-control:development";
}
/** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
const built: string[] = [];
src/suite.ts
+3 -3
View File
@@ -53,9 +53,9 @@ export async function runSuite(args: string[]): Promise<number> {
// the long run reaches the end of that path in about 160 seconds.
//
// So it cost a whole scenario, every passing run, to save about 45 seconds on a failing one.
// A scenario is three machines including a registry that boots a kernel to serve files, which
// is where the two minutes went. `test/integration/canary.test.ts` is still there and still
// runs when it is named; it is no longer raised on the way to everything else.
// A scenario is machines that each boot a kernel, which is where the two minutes went.
// `test/integration/canary.test.ts` is still there and still runs when it is named; it is no
// longer raised on the way to everything else.
const { code, seen } = await runFiles(files);
console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files, process.env, against)));
src/warm.ts
+11 -10
View File
@@ -23,6 +23,7 @@ import { homedir } from "node:os";
import { list, restore, snapshot, snapshots, destroy } from "./lifecycle/operate.ts";
import type { Against } from "./lastrun.ts";
import { whatWasTested } from "./lastrun.ts";
import type { HeldImage } from "./pinning.ts";
/** The state a warm instance is kept at. One label, because a second is a state nobody named. */
export const label = "warm";
@@ -31,13 +32,13 @@ export interface Warm {
scenario: string;
instanceId: string;
/**
* The image references the scenario's registry serves, pinned by digest.
* The mesh's own images, as loaded onto this instance's machines, by the ID each is held under.
*
* Kept because they are worked out while raising and a restored instance never raises. Without
* them a warm run knows nothing about what it can pull, and every test naming an image fails
* for a reason that has nothing to do with what it was testing.
* them a warm run knows nothing about what its machines hold, and every test naming one of our
* images fails for a reason that has nothing to do with what it was testing.
*/
images: string[];
images: HeldImage[];
/** The commit each repository was at when this was brought to its state. */
against: Against;
at: string;
@@ -187,23 +188,23 @@ export async function cool(): Promise<string | null> {
}
/**
* What a raised scenario stocked, held until it is kept.
* What a raised scenario loaded onto its machines, held until it is kept.
*
* Raising works the images out and snapshotting happens later, so this carries them between the
* two without the caller having to hold them.
*/
const stock = new Map<string, string[]>();
const stock = new Map<string, HeldImage[]>();
export function rememberStock(instanceId: string, images: string[]): void {
export function rememberStock(instanceId: string, images: HeldImage[]): void {
stock.set(instanceId, images);
}
function stockOf(instanceId: string): string[] {
function stockOf(instanceId: string): HeldImage[] {
return stock.get(instanceId) ?? [];
}
/** What a restored instance's registry serves, from when it was warmed. */
export function warmStock(instanceId: string): { images: string[] } {
/** What a restored instance's machines hold, from when it was warmed. */
export function warmStock(instanceId: string): { images: HeldImage[] } {
const warm = remembered();
if (!warm || warm.instanceId !== instanceId) return { images: [] };
return { images: warm.images };
test/diagram.test.ts
+1 -1
View File
@@ -109,7 +109,7 @@ test("segments are ordered public first, then by depth behind them", () => {
});
test("a declared address appears on the machine that holds it", () => {
assert.match(xml, /192\.168\.1\.135/);
assert.match(xml, /10\.99\.1\.135/);
assert.match(xml, /198\.51\.100\.7/);
});
test/egress-routes.test.ts
+128
View File
@@ -0,0 +1,128 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { parseScenario } from "../src/declaration/parse.ts";
import { scenarioRoutesFor } from "../src/lifecycle/address.ts";
/**
* The uplink and the declared gateway must not fight.
*
* **This is the one decision the registry's removal turned on, and it is invisible in a raise.**
* Every machine that needs an image now has an `egress` uplink, and the uplink's DHCP offers a
* default route. So did the scenario: a machine behind a household gateway defaulted through it, a
* machine on a public segment defaulted through transit. Both of those are containers that reach
* the scenario and nothing else — no route to the real internet, by design, because they exist to
* reproduce a household router rather than to be one.
*
* A default route through either is therefore a black hole for anything outside, and it beats the
* uplink's route on metric. The machine would sit failing every pull with a routing table that
* looks perfectly reasonable.
*
* The answer is that an egress machine states the scenario's ranges explicitly and lets the uplink
* be the default. These tests are how that is checked without spending an hour raising four nodes.
*/
const HOUSEHOLD = `
scenario: household
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
home:
kind: private
cidr: [10.99.1.0/24]
gateway:
to: hosting
address: [192.0.2.50]
nat: [v4]
forwardable: true
machines:
novox:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
ace:
at: { segment: home, address: [10.99.1.10] }
egress: true
sealed:
at: { segment: home, address: [10.99.1.99] }
`;
test("a machine behind a gateway still reaches the scenario through that gateway", () => {
// The whole point of the topology: home→public is a masqueraded outbound path, and the overlay
// handshake has to survive it. An egress machine that stopped using its gateway would be
// testing a flat network with extra steps.
const routes = scenarioRoutesFor(parseScenario(HOUSEHOLD), "ace");
assert.deepEqual(routes, [{ cidr: "192.0.2.0/24", via: "10.99.1.1" }]);
});
test("a machine's own segment gets no route — it is already on-link", () => {
const routes = scenarioRoutesFor(parseScenario(HOUSEHOLD), "ace");
assert.ok(!routes.some((r) => r.cidr === "10.99.1.0/24"), JSON.stringify(routes));
});
/**
* **The dangerous one.** `home` is 10.99.1.0/24 — a documentation range in spirit, an ordinary
* private one in fact, and very possibly the network the workstation itself is on.
*
* With one public segment there is no transit router, so novox has no path to `home` at all. Left
* to fall through, that traffic would leave by the uplink and land on whatever the workstation can
* reach. Unreachable is both the faithful reproduction of what it had before — a default route into
* scenery that dropped it — and the only safe answer.
*/
test("a range with no path inside the scenario is unreachable, not leaked to the uplink", () => {
const routes = scenarioRoutesFor(parseScenario(HOUSEHOLD), "novox");
assert.deepEqual(routes, [{ cidr: "10.99.1.0/24", via: null }]);
});
test("a machine without egress is left to its default route, and states nothing", () => {
// Not because it needs no routes — it has one, a default through its gateway, applied the old
// way. This function is only asked about machines whose default belongs to the uplink.
const scenario = parseScenario(HOUSEHOLD);
assert.equal(scenario.machines["sealed"]?.egress, undefined);
});
const TWO_PUBLIC = `
scenario: two-public
segments:
hosting:
kind: public
cidr: [192.0.2.0/24]
elsewhere:
kind: public
cidr: [198.51.100.0/24]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
`;
test("with a second public segment the transit router is the way across, as it always was", () => {
// Transit is raised only when there is more than one public segment, so this is exactly the
// case where pointing at it means something.
const routes = scenarioRoutesFor(parseScenario(TWO_PUBLIC), "anchor");
assert.deepEqual(routes, [{ cidr: "198.51.100.0/24", via: "192.0.2.254" }]);
});
const V6 = `
scenario: both-families
segments:
hosting:
kind: public
cidr: [192.0.2.0/24, "2001:db8:a::/48"]
elsewhere:
kind: public
cidr: [198.51.100.0/24, "2001:db8:b::/48"]
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10, "2001:db8:a::10"] }
egress: true
`;
test("each family is routed through its own next hop", () => {
// A v6 range routed via a v4 next hop is not a route, and the reverse is not either.
const routes = scenarioRoutesFor(parseScenario(V6), "anchor");
assert.deepEqual(routes, [
{ cidr: "198.51.100.0/24", via: "192.0.2.254" },
{ cidr: "2001:db8:b::/48", via: "2001:db8:a::fffe" },
]);
});
test/integration/anthropic-bed.test.ts
+9 -15
View File
@@ -49,7 +49,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -73,7 +74,7 @@ const ACCESS_TOKEN = "at-lab-access-token-minted-by-the-stub";
const ROTATED_REFRESH = "rt-lab-rotated-still-only-the-manager";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -113,20 +114,13 @@ async function meshTry(command: string): Promise<{ out: string; ok: boolean }> {
return on(`docker exec mesh-control /mesh-control ${command}`);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -177,7 +171,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/assigned-audit.test.ts
+14 -20
View File
@@ -7,7 +7,7 @@
* container that connects over amqps with that account — never the broker's own. The trail filling
* is the proof the delivered, scoped credential authenticated and the subscription bound.
*
* It needs the host binary, the substrate bundle, and the runtime image stocked by the scenario:
* It needs the host binary, the substrate bundle, and the runtime image the scenario loads:
*
* MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
@@ -22,7 +22,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -40,8 +41,8 @@ const SCENARIO = "audit-node";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -67,22 +68,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -125,7 +119,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
@@ -151,7 +145,7 @@ after(async () => {
test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", {
skip, timeout: 900_000,
}, async () => {
// The assigned-module manifest (mesh-catalog), its runtime image the digest this registry serves.
// The assigned-module manifest (mesh-catalog), its runtime image the ID the machine holds.
const manifest = JSON.stringify({
module: "audit-logger",
version: "1",
test/integration/assigned-catalogue-apps.test.ts
+12 -18
View File
@@ -30,7 +30,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -48,8 +49,8 @@ const SCENARIO = "catalogue-apps";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -75,22 +76,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -133,7 +127,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-catalogue-media.test.ts
+14 -19
View File
@@ -26,7 +26,8 @@
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
* scripts/build-module-runtime.sh {sonarr,radarr} build the runtime images into the local daemon;
* scenarios/catalogue-media.yml stocks them. lscr.io/linuxserver/{sonarr,radarr} must be in the
* local daemon to be stocked. Each *arr runtime is given a lab API key so its client constructs and
* local daemon; the service images are pulled from the internet. Each *arr runtime is given a lab
* API key so its client constructs and
* its tools register (as plex is given a lab token) — the server need not be configured by hand.
*/
@@ -37,7 +38,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -55,8 +57,8 @@ const SCENARIO = "catalogue-media";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -82,22 +84,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -140,7 +135,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-catalogue-mqtt.test.ts
+12 -18
View File
@@ -26,7 +26,7 @@
* scripts/build-module-runtime.sh mosquitto builds mesh-runtime-mosquitto:development (carrying
* mosquitto_ctrl and the compiled bootstrap entrypoint) into the local daemon, which
* scenarios/catalogue-mqtt.yml stocks. eclipse-mosquitto:2 must be in the local daemon to be
* stocked; the host pulls both from the scenario's own registry by digest.
* the host pulls both from the internet over its uplink, by the digests the catalogue pins.
*/
import { test, before, after } from "node:test";
@@ -36,7 +36,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -54,7 +55,7 @@ const SCENARIO = "catalogue-mqtt";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -80,22 +81,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -138,7 +132,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-catalogue-small.test.ts
+13 -19
View File
@@ -21,7 +21,7 @@
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
* scripts/build-module-runtime.sh {postgres,redis,minio,plex} build the four runtime images into the
* local daemon; scenarios/catalogue-small.yml stocks them. postgres:17-alpine, redis:7-alpine and
* minio/minio:latest must be in the local daemon to be stocked.
* minio/minio:latest is pulled from the internet by the node itself.
*/
import { test, before, after } from "node:test";
@@ -31,7 +31,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -49,8 +50,8 @@ const SCENARIO = "catalogue-small";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -76,22 +77,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -140,7 +134,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-grafana.test.ts
+9 -15
View File
@@ -22,7 +22,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -40,7 +41,7 @@ const SCENARIO = "grafana-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -65,20 +66,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -121,7 +115,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/assigned-model-usage.test.ts
+12 -18
View File
@@ -38,7 +38,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -58,8 +59,8 @@ const SCENARIO = "model-usage-bed";
const NODE = "laptop";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -85,22 +86,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -164,7 +158,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/assigned-plex.test.ts
+13 -19
View File
@@ -10,7 +10,7 @@
* proof the invocation routed to the assigned runtime, ran plex's real code, and replied, all under
* the scoped account and never the broker's own.
*
* It needs the host binary, the substrate bundle, and the runtime image stocked by the scenario:
* It needs the host binary, the substrate bundle, and the runtime image the scenario loads:
*
* MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
@@ -25,7 +25,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -43,8 +44,8 @@ const SCENARIO = "plex-node";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -70,22 +71,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -128,7 +122,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-redis.test.ts
+10 -16
View File
@@ -12,7 +12,7 @@
* has no way yet to deliver one to a provider's runtime (04-ISSUES). The manifest here sets a
* lab-local key so the mechanism can be proven; the delivery is a separate, open design question.
*
* It needs the host binary, the substrate bundle, and the runtime image stocked by the scenario:
* It needs the host binary, the substrate bundle, and the runtime image the scenario loads:
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
* scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development into the local
@@ -26,7 +26,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -44,7 +45,7 @@ const SCENARIO = "redis-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -69,20 +70,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -125,7 +119,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/assigned-schedule-tick.test.ts
+12 -18
View File
@@ -38,7 +38,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -56,8 +57,8 @@ const SCENARIO = "schedule-tick";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -83,22 +84,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -155,7 +149,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-sonarr.test.ts
+9 -15
View File
@@ -20,7 +20,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -38,7 +39,7 @@ const SCENARIO = "sonarr-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -63,20 +64,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -119,7 +113,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/assigned-tools-confluence.test.ts
+12 -18
View File
@@ -29,7 +29,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -47,8 +48,8 @@ const SCENARIO = "tools-confluence";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -74,22 +75,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -132,7 +126,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-tools-gitlab.test.ts
+12 -18
View File
@@ -28,7 +28,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -46,8 +47,8 @@ const SCENARIO = "tools-gitlab";
const MACHINE = "anchor";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -73,22 +74,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -131,7 +125,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/assigned-two-node-db.test.ts
+12 -18
View File
@@ -44,7 +44,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -63,8 +64,8 @@ const SCENARIO = "two-node-db";
const NODE = "laptop";
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -90,22 +91,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -173,7 +167,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// The first node raises the substrate — store, broker, control — from the bundle its host carries,
// its digests rewritten to the ones this scenario's own registry serves.
test/integration/builds.test.ts
+35 -14
View File
@@ -20,7 +20,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
@@ -42,7 +43,22 @@ const skip = !capability.usable
const SCENARIO = "first-node";
const MACHINE = "anchor";
let instanceId = "";
let registry = "";
/**
* Where a build publishes to.
*
* **The mesh has a registry, and this is that one.** `mesh-catalog/modules/registry` serves the
* mesh's artifact store on port 5000; a build publishes into it. This test starts the same image
* on the machine directly rather than assigning the module, because what is under test is the
* build chain and not module delivery.
*
* It used to publish into the registry the LAB raised inside the scenario — scenery pretending to
* be upstream, which is the thing this change removed. A registry the mesh runs and a registry the
* lab runs are different claims, and only the first exists in production.
*/
const ARTIFACT_STORE =
"registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
const registry = "127.0.0.1:5000";
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -66,28 +82,33 @@ async function mesh(command: string): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`);
}
/** The bundle, pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const pinned of images) {
const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), pinned);
}
return text;
/** The bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
before(async () => {
if (skip) return;
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
instanceId = raised.instanceId;
const first = raised.images[0];
assert.ok(first, "the scenario stocked no images, so there is no registry to publish to");
registry = first.slice(0, first.indexOf("/"));
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`);
// The mesh's artifact store, standing where the `registry` module would. Read back rather than
// assumed: a builder publishing into a registry that never came up fails several minutes later,
// as a manifest naming a blob nobody has.
await must(
`docker run -d --name mesh-registry --restart unless-stopped ` +
`-p ${registry}:5000 ${ARTIFACT_STORE}`,
);
let serving = false;
for (let i = 0; i < 30 && !serving; i++) {
({ ok: serving } = await on(`curl -sf http://${registry}/v2/ >/dev/null`));
if (!serving) await new Promise((r) => setTimeout(r, 2_000));
}
assert.ok(serving, "the mesh's artifact store never answered, so a build has nowhere to publish");
// A module repository on the machine. Local rather than fetched, because what is under test is
// the mesh's chain and not whether the lab can reach a forge.
await must(`mkdir -p /root/shell/files`);
test/integration/certificates.test.ts
+9 -2
View File
@@ -32,6 +32,14 @@ const SCENARIO = "a-public-name";
const MACHINE = "anchor";
const NAME = "photos.example";
const ACME = "/var/lib/acme";
/**
* The ACME server under test, pulled by the machine over its uplink.
*
* It used to be served from a registry the lab raised inside the scenario. Nothing outside the lab
* has one, so an image only reachable there was a fiction — and this test is about a certificate
* being obtained over a real path.
*/
const AUTHORITY = "ghcr.io/letsencrypt/pebble:2.5.0";
let instanceId = "";
@@ -59,8 +67,7 @@ before(async () => {
const instance = await raise(scenario, {});
instanceId = instance.instanceId;
const pebble = instance.images.find((r) => r.includes("pebble"));
assert.ok(pebble, `the scenario stocked no ACME server: ${instance.images.join(", ")}`);
const pebble = AUTHORITY;
await must(`mkdir -p ${ACME}/cache`);
test/integration/events.test.ts
+6 -13
View File
@@ -33,7 +33,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
@@ -97,17 +98,8 @@ async function must(command: string, timeoutMs?: number): Promise<string> {
* is not this one; matching by repository and rewriting to the digest this registry assigned is
* what makes it applicable (the same rewrite mesh.test.ts does).
*/
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const pinned of images) {
const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(
new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"),
pinned,
);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
/** Read the trail back as parsed JSON lines. */
@@ -129,7 +121,8 @@ before(async () => {
instanceId = raised.instanceId;
// The node raises its substrate — store, broker and the rest — from the bundle, applied from a
// file because the digests are this registry's and are not known until it is up.
// file because the control plane's image is named by the ID this machine holds it under,
// which is not knowable until it has been handed over.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/harness.ts
+115
View File
@@ -9,11 +9,126 @@
*/
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
import { destroy, list } from "../../src/lifecycle/operate.ts";
import { diagramFromLive } from "../../src/diagram/from-live.ts";
import { duplicateAddresses, describeConflicts, type Held } from "../../src/lifecycle/invariants.ts";
import type { Scenario } from "../../src/declaration/types.ts";
import { mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, type HeldImage } from "../../src/pinning.ts";
// --- the substrate bundle, and what its three images are on a real machine ---------------------
/**
* The example bundle in mesh-host names a registry that no longer exists.
*
* `examples/substrate-first-node.lock` was written **for a target**, and the target was the lab: it
* pins `192.0.2.250:5000/…` because that is where the registry the lab used to raise served from.
* That registry is gone, so those three references name nothing.
*
* Two of them are ordinary third-party images and belong to the internet. Rather than invent
* digests here, they are the ones the mesh's own modules already pin — mesh-catalog's `postgres`
* and `lavinmq` — so the substrate's store and broker are literally the images the mesh runs. The
* third, mesh-control, exists in no registry at all and becomes the ID the machine holds it under.
*
* **The bundle itself should be fixed in mesh-host**, and this substitution deleted with it. It is
* here because the file lives in another repository and because a fixture that lies about where an
* image comes from is exactly what this change is removing.
*/
const UPSTREAM_STORE =
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
const UPSTREAM_BROKER =
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
/**
* The substrate bundle as a machine should receive it.
*
* Third-party references become upstream ones, which the machine pulls over its uplink; ours
* become the ID the machine was handed. Nothing points inside the scenario any more, which is the
* whole of this change: what the bed proves about a bootstrap is now what would happen anywhere.
*/
export function substrateBundle(path: string, held: HeldImage[]): string {
let text = readFileSync(path, "utf8");
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
text = text.replaceAll(
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
return pinnedInto(text, held);
}
/**
* The upstream reference for a third-party image, as the mesh's own catalogue pins it.
*
* A bed that writes a manifest by hand still has to name an image exactly — mesh-host refuses a
* tag, and rightly (novox/hq ADR 0006). While the lab had a registry the beds sidestepped that by
* naming a repository and letting the rewrite supply a digest; there is nothing to supply one now,
* so the digest has to be written down.
*
* These are the digests mesh-catalog's own modules pin, taken from `mesh-catalog/modules/*` — so a
* bed runs the image the mesh runs, and a bed that drifts from the catalogue is a bed testing a
* different postgres than the mesh ships.
*/
const UPSTREAM = new Map<string, string>([
["alpine", "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"],
["baserow/baserow", "baserow/baserow@sha256:834424a10413798567f76428f255dc259445b7f8dcec56598c05b4073bb2a124"],
["cloudamqp/lavinmq", "cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b"],
["eclipse-mosquitto", "eclipse-mosquitto@sha256:6f8d8a947c506f8a2290ec65cd4bd2bc7cb4d43fb5f6271f861cb013e2ef9797"],
["ghcr.io/umami-software/umami", "ghcr.io/umami-software/umami@sha256:fa32d116cf20cad52cbc3fad9a63b46e7fa02299d8f967168eb453d49c476b4a"],
["letta/letta", "letta/letta@sha256:1d2e0692514287c5ed1a483e14e16ed945f8632d315539f5e66373bb7d7c471b"],
["lscr.io/linuxserver/radarr", "lscr.io/linuxserver/radarr@sha256:119aaa4a4f7349bcd2a136c5373a0d7925b5479915c7dfe0c0ad352db2a6d438"],
["lscr.io/linuxserver/sonarr", "lscr.io/linuxserver/sonarr@sha256:c19aa4ecdf03d73e1d5c901da33744cb7eb4d921f89bafed1ca264601d7fa224"],
["lscr.io/linuxserver/unifi-controller", "lscr.io/linuxserver/unifi-controller@sha256:fcd5d8b13a77a588c79c1b49e5fc9ad08115aa3bb1a3576c589c64908a68845f"],
["minio/minio", "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2"],
["mongo", "mongo@sha256:e3fa459b4f4b72f3257c67a23c145e250b8b5700f033860392c68539b998bbe3"],
["ollama/ollama", "ollama/ollama@sha256:32931b46719f673c05fdbaa81ccb26da18ea4a1c57590a754874ab28ba269eb2"],
["postgres", "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"],
["redis", "redis@sha256:ff02b58f971e7d7d156a1267e283fcbbeee91773b6aa36c49dac28ecfe28eadf"],
["registry", "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"],
["synesthesiam/marytts", "synesthesiam/marytts@sha256:45970ecb3e21a2981c66c60563a70cf00be8e95c02565e7d74b3a73dcec7db2c"],
]);
/**
* What a manifest's image reference becomes on the machine.
*
* Four cases, and the second one is the whole change:
*
* - **Ours** becomes the ID the machine holds it under. Nothing serves it, and nothing needs to.
* - **Anything already pinned by digest** is returned exactly as written. The machine pulls it
* from the internet, over its uplink, which is what a real machine does and what the lab spent
* a long time serving from a registry of its own instead.
* - **A bare repository a bed names by hand** is given the digest mesh-catalog pins for it, so a
* bed runs the image the mesh ships. A tag would be refused by mesh-host anyway.
* - **A tag this harness has never heard of** is passed through untouched, and said out loud.
*
* That last case is not politeness, it is a finding the lab's registry was hiding. Seven catalogue
* modules name `registry-api.…/novox/…:latest` — a TAG, which ADR 0006 forbids and mesh-host
* refuses. It never showed, because the rewrite replaced every reference with a digest the lab's
* registry had assigned, tag or not. There is nothing to replace it with now, and the honest
* outcome is that those modules fail to apply, saying exactly why, on the node that carries them —
* rather than an assertion here taking the whole bed down before it starts.
*/
export function onTheMachine(reference: string, held: HeldImage[]): string {
if (mustBeHandedOver(reference)) {
const found = referenceFor(held, repositoryOf(reference));
assert.ok(
found,
`nothing loaded ${reference} onto the machines. They hold:\n ` +
held.map((i) => `${i.repository} ${i.reference}`).join("\n "),
);
return found;
}
if (reference.includes("@sha256:")) return reference;
const upstream = UPSTREAM.get(repositoryOf(reference));
if (upstream) return upstream;
console.log(
`UNPINNED: ${reference} names a tag, not a digest. The host will refuse it (novox/hq ` +
`ADR 0006). The lab's own registry used to paper over this by assigning a digest to ` +
`whatever was pushed; nothing does now. Fix the manifest, or add its digest to the ` +
`harness's UPSTREAM table.`,
);
return reference;
}
export interface Capability {
usable: boolean;
test/integration/lavinmq-bed.test.ts
+11 -17
View File
@@ -41,7 +41,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -62,7 +63,7 @@ const NODE = "laptop";
const CONSUMER_LOGIN = "mesh_laptop_ping";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -88,22 +89,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -161,7 +155,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/local-model-bed.test.ts
+9 -15
View File
@@ -26,7 +26,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -44,7 +45,7 @@ const SCENARIO = "local-model-bed";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -73,20 +74,13 @@ async function meshTry(command: string): Promise<{ out: string; ok: boolean }> {
return on(`docker exec mesh-control /mesh-control ${command}`);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -136,7 +130,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/mesh-grant-end-to-end.test.ts
+9 -15
View File
@@ -23,7 +23,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -41,7 +42,7 @@ const SCENARIO = "redis-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -66,20 +67,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -122,7 +116,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/mesh.test.ts
+42 -49
View File
@@ -21,10 +21,10 @@ import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { pinnedInto, stillUnpinned } from "../../src/pinning.ts";
import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
@@ -49,23 +49,27 @@ const skip = !capability.usable
const SCENARIO = "two-nodes";
let instanceId = "";
/** The scenario's own registry, which serves the images a module may mirror. */
let registry = "";
/** What that registry actually serves, by repository. */
let stocked: string[] = [];
/**
* The pinned reference for one of the scenario's images.
* The MESH's own artifact store, once the registry module is running on the anchor.
*
* By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and
* asking for it fails with "not found", which reads like a missing image rather than a naming
* convention. A digest is also what a declaration pins, so this is the reference a module would
* really carry.
* Not a registry the lab raised — there is no longer any such thing. A build publishes into the
* store the mesh itself runs, which is the only registry that exists outside this repository.
*/
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
const registry = "127.0.0.1:5000";
/**
* The registry module's image, pinned upstream, pulled by the machine over its uplink.
*
* The digest mesh-catalog's `registry` module pins, so the store the mesh runs here is the store
* the mesh runs anywhere.
*/
const ARTIFACT_STORE =
"registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function quote(s: string): string {
@@ -179,23 +183,14 @@ async function settled(node: string, withinMs = 480_000): Promise<void> {
}
/**
* The bundle, with every image reference pointed at this scenario's registry.
* The bundle, as a machine should receive it.
*
* Matched by repository rather than by the whole reference, because the address and the digest
* both differ from whatever the committed bundle names — and a bundle that names the wrong
* registry is not wrong, it is built for a different target.
* The committed example was written for a target that had a registry the lab raised. Its two
* third-party images become upstream references the machine pulls itself; mesh-control, which
* exists in no registry, becomes the ID this machine was handed.
*/
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const pinned of images) {
const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(
new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"),
pinned,
);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
@@ -219,7 +214,7 @@ before(async () => {
if (said.use === "restore") {
instanceId = said.instanceId;
const seconds = await returnTo(instanceId);
stocked = warmStock(instanceId).images;
held = warmStock(instanceId).images;
// **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything
// this suite started by hand is gone — the host most of all. Without it the mesh looks
@@ -255,13 +250,11 @@ before(async () => {
instanceId = raised.instanceId;
// The first node raises everything from a file rather than from a bundle built into the binary,
// because the digests are this registry's and are not known until it is up.
// because the control plane's image is named by the ID this machine holds it under, which is not
// knowable until it has been handed over.
held = raised.images;
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`);
stocked = raised.images;
const first = raised.images[0];
assert.ok(first, "the scenario stocked no images, so nothing can be mirrored");
registry = first.slice(0, first.indexOf("/"));
// A build machine, so anything here can ask the mesh to build something. Placed rather than
// assumed: nothing else in this scenario would start one.
@@ -279,7 +272,7 @@ before(async () => {
if (warming) {
// Snapshotted only now, with everything up: a state worth returning to is the one after the
// part nobody wants to repeat.
await rememberStock(instanceId, stocked);
await rememberStock(instanceId, held);
const warm = await keep(SCENARIO, instanceId);
console.log(`warm: ${warm.instanceId} kept, against ` +
Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
@@ -608,13 +601,13 @@ test("a machine that fell behind catches up without being named", { skip, timeou
});
test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async () => {
// Artifacts go to a registry, and the only registries that existed were raised by the lab or by
// the bootstrap bundle. A mesh had no way to run its own.
// Artifacts go to a registry, and a mesh had no way to run its own — the only one that existed
// was raised by the lab, which is to say it existed nowhere but here.
//
// **Named, not mirrored** (novox/hq 04-ISSUES/029). Mirroring publishes to the artifact store,
// and the builder will not start without one — so a module that provides the store and builds
// its own image asks the mesh to put an artifact into the thing that artifact is needed to
// create. It worked here only because the scenario's registry was already standing to receive
// create. It used to pass here only because the LAB's registry was already standing to receive
// the push, which is exactly why a real first mesh would have found this and the lab did not.
//
// So the image is named by digest, the way the bundle names the three a first node starts from.
@@ -626,7 +619,7 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
`"serves":{"artifact-store":{"port":5000}},` +
`"resources":[` +
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
`{"id":"store","type":"container","name":"mesh-registry","image":"${pinned("registry")}",` +
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
`"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
`> /root/registry/module.json`);
// **Added, not built** — and this is the half that proves the fix. Building needs a builder,
@@ -677,7 +670,7 @@ test("a machine serves its internal name with a certificate the mesh issued", {
// The name it was issued for is the one the mesh gave this machine.
const named = await must("anchor",
`openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
`docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` +
`docker run --rm -v /etc/mesh:/m ${ARTIFACT_STORE} sh -c ` +
`"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);
@@ -1086,7 +1079,7 @@ test("a route is a grant: a workload is reached by the name it asked for", {
`"listens":[{"port":8088,"from":"mesh","why":"the proxy reaches it here"}],` +
`"resources":[{"id":"dir","type":"directory","path":"/etc/storefront","mode":"0755"},` +
`{"id":"app","type":"container","name":"storefront",` +
`"image":"${pinned("registry")}","ports":["8088:5000"]}]}' > /tmp/storefront.json`);
`"image":"${ARTIFACT_STORE}","ports":["8088:5000"]}]}' > /tmp/storefront.json`);
for (const f of ["frontdoor", "storefront"]) {
await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
await mesh(`module add /${f}.json`);
@@ -1456,7 +1449,7 @@ test("a container reaches another machine by the name the mesh gave it", {
await must("anchor", `printf %s '{"module":"resolves","version":"1",` +
`"capabilities":["container-runtime"],` +
`"resources":[{"id":"idle","type":"container","name":"resolves",` +
`"image":"${pinned("registry")}"}]}' > /tmp/resolves.json`);
`"image":"${ARTIFACT_STORE}"}]}' > /tmp/resolves.json`);
await must("anchor", `docker cp /tmp/resolves.json mesh-control:/resolves.json`);
await mesh("module add /resolves.json");
await mesh("assign laptop resolves");
@@ -1810,7 +1803,7 @@ test("the real modules resolve together, and compose a declaration a host accept
// until it is built — so the file legitimately carries a placeholder, and composing a
// declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is
// what this test used to do.
const pinned = pinnedInto(raw, stocked);
const pinned = pinnedInto(raw, held);
// What this scenario does not serve cannot be redirected, and a module still naming a
// placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped
// and said, rather than silently dropped: a planning test quietly covering four modules
@@ -1934,8 +1927,8 @@ test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000
for (const name of ["postgres", "gitea"]) {
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
// An image the mesh builds has no digest until it is built, and one it does not build belongs
// to whichever registry served it. Both are answered by this scenario's own registry.
const pinned = pinnedInto(raw, stocked);
// to whichever registry served it. Only the first is rewritten; the second is pulled.
const pinned = pinnedInto(raw, held);
assert.deepEqual(stillUnpinned(pinned), [],
`${name} still names an image nothing serves, so it could not start`);
await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`);
@@ -2021,7 +2014,7 @@ test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600
// keyspace, so the grant is a pattern — and the test is that the pattern means what the
// manifest said, in both directions.
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8");
const pinned = pinnedInto(raw, stocked);
const pinned = pinnedInto(raw, held);
assert.deepEqual(stillUnpinned(pinned), [],
"redis still names an image nothing serves, so it could not start");
await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`);
test/integration/minio-grant-end-to-end.test.ts
+9 -15
View File
@@ -21,7 +21,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -42,7 +43,7 @@ const SCENARIO = "minio-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -67,20 +68,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -129,7 +123,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/objectstore.test.ts
+13 -13
View File
@@ -45,8 +45,16 @@ const ROOT_PASSWORD_FILE = "/var/lib/objectstore/root.secret";
const ENDPOINT = "http://127.0.0.1:9000";
let instanceId = "";
/** The store's image, by digest, from the registry the scenario raised. */
let storeImage = "";
/**
* The store and the vendor's client, pinned upstream and pulled by the machine over its uplink.
*
* The store is the digest the mesh's own minio module pins, so this is the store the mesh runs.
* Both used to come from a registry the lab raised inside the scenario; no production mesh has
* one, so a test that could only fetch from it was proving something about the lab.
*/
const storeImage =
"minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2";
const clientImage = "minio/mc:RELEASE.2025-08-13T08-35-41Z";
function shellQuote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -141,18 +149,10 @@ before(async () => {
const instance = await raise(scenario, {});
instanceId = instance.instanceId;
// From the registry the scenario raised, by digest. There is no route to a public registry from
// a documentation range, which is the point of the lab having its own.
const store = instance.images.find((r) => r.includes("minio/minio"));
const client = instance.images.find((r) => r.includes("minio/mc"));
assert.ok(store, `the scenario stocked no store image: ${instance.images.join(", ")}`);
assert.ok(client, `the scenario stocked no client image: ${instance.images.join(", ")}`);
storeImage = store;
// The client, taken out of the vendor's own image onto the machine. The provisioner drives it,
// so it has to be here — and taking it from the stocked image is what keeps this test off any
// public network.
await must(`docker create --name mc-source ${client}`);
// so it has to be here. The machine pulls the image itself, over its uplink, the way it pulls
// everything third-party.
await must(`docker create --name mc-source ${clientImage}`);
await must(`docker cp mc-source:/usr/bin/mc /usr/local/bin/mc && chmod 755 /usr/local/bin/mc`);
await must(`docker rm mc-source`);
test/integration/openai-bed.test.ts
+9 -15
View File
@@ -24,7 +24,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -46,7 +47,7 @@ const MACHINE = "anchor";
const API_KEY = "sk-lab-openai-static-key-value-for-the-bed-only";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -75,20 +76,13 @@ async function meshTry(command: string): Promise<{ out: string; ok: boolean }> {
return on(`docker exec mesh-control /mesh-control ${command}`);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -139,7 +133,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/postgres-grant-end-to-end.test.ts
+9 -15
View File
@@ -21,7 +21,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -39,7 +40,7 @@ const SCENARIO = "postgres-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -64,20 +65,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -120,7 +114,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/provider-on-backend-network.test.ts
+9 -15
View File
@@ -20,7 +20,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -38,7 +39,7 @@ const SCENARIO = "redis-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -63,20 +64,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -119,7 +113,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/provider-uses-mesh-credential.test.ts
+9 -15
View File
@@ -24,7 +24,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -42,7 +43,7 @@ const SCENARIO = "redis-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -67,20 +68,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -123,7 +117,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/provisioner.test.ts
+9 -8
View File
@@ -34,8 +34,15 @@ const GRANTS = "/var/lib/postgres/grants";
const SUPER = "postgres://postgres:super@127.0.0.1:5432/postgres?sslmode=disable";
let instanceId = "";
/** The postgres image, by digest, from the registry the scenario raised. */
let image = "";
/**
* The database, pinned upstream and pulled by the machine over its uplink.
*
* The same digest the mesh's own postgres module pins, so this is the database the mesh runs
* rather than a lookalike. It used to come from a registry the lab raised inside the scenario;
* nothing outside the lab has one, so what that proved about fetching an image was true only here.
*/
const image =
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
function shellQuote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -142,12 +149,6 @@ before(async () => {
const instance = await raise(scenario, {});
instanceId = instance.instanceId;
// From the registry the scenario raised, by digest. There is no route to a public registry from
// a documentation range, which is the point of the lab having its own.
const stocked = instance.images.find((r) => r.includes("postgres"));
assert.ok(stocked, `the scenario stocked no postgres image: ${instance.images.join(", ")}`);
image = stocked;
await must(
`docker run -d --name mesh-db -e POSTGRES_PASSWORD=super ` +
`-p 127.0.0.1:5432:5432 ${image}`,
test/integration/route-forwarding.test.ts
+11 -17
View File
@@ -37,7 +37,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -57,7 +58,7 @@ const NAME = "hello.example";
const PAGE = "hello from hello-web, routed by the mesh";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -83,22 +84,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The pinned reference for one of the scenario's images, by repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** The reference a manifest should carry, once this scenario has been raised. */
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -147,7 +141,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// Raise the substrate — store, broker, control — from the bundle.
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/integration/runtime-restart-on-config.test.ts
+9 -15
View File
@@ -24,7 +24,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -42,7 +43,7 @@ const SCENARIO = "grafana-node";
const MACHINE = "anchor";
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -67,20 +68,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function tokenFrom(said: string): string {
@@ -133,7 +127,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
test/integration/underlay.test.ts
+3 -3
View File
@@ -45,7 +45,7 @@ test("ADR 0016 — the lab provides the underlay and NOTHING of the overlay", {
test("ADR 0016 — the declared address IS what the machine holds", { skip }, async () => {
const { stdout } = await exec(instanceId, "home-server", ["ip", "-o", "-4", "addr", "show"]);
assert.match(stdout, /192\.168\.1\.135\/24/);
assert.match(stdout, /10\.99\.1\.135\/24/);
});
test("design — raise waits for USABLE, not for the call to return", { skip, timeout: 120_000 }, async () => {
@@ -75,7 +75,7 @@ test("ADR 0016 — a router is scenery: containers, while machines are virtual m
test("design — NAT: a private address is not reachable from outside", { skip, timeout: 120_000 }, async () => {
const { stdout } = await exec(instanceId, "anchor", [
"sh", "-c", "ping -c1 -W2 192.168.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable",
"sh", "-c", "ping -c1 -W2 10.99.1.135 >/dev/null 2>&1 && echo reachable || echo unreachable",
]);
assert.equal(stdout.trim(), "unreachable");
});
@@ -139,7 +139,7 @@ test("the live diagram reads the hypervisor, and a VM's addresses are not lost",
assert.ok(server, "home-server missing from the live picture");
assert.equal(server.kind, "machine");
assert.ok(
server.attachments.some((a) => a.addresses.some((address) => address.startsWith("192.168.1.135"))),
server.attachments.some((a) => a.addresses.some((address) => address.startsWith("10.99.1.135"))),
`a virtual machine's addresses were not read back: ${JSON.stringify(server.attachments)}`,
);
});
test/integration/whole-mesh-ace.test.ts
+11 -24
View File
@@ -17,7 +17,7 @@
* apps unifi portainer
*
* Each committed module.json is LOADED from mesh-catalog (not hand-written); its container image
* references are rewritten to what this scenario's own registry serves by digest, and the co-located
* references of OURS are rewritten to the IDs the machine holds, and the co-located
* host-port collisions are remapped at load time (see REMAP).
*
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
@@ -31,7 +31,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -148,7 +149,7 @@ const REMAP: Record<string, Record<string, string>> = {
};
let instanceId = "";
let stocked: string[] = [];
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -173,27 +174,13 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
function repositoryFor(reference: string): string {
const withoutDigest = reference.split("@")[0] ?? reference;
const lastColon = withoutDigest.lastIndexOf(":");
const lastSlash = withoutDigest.lastIndexOf("/");
return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`);
return found;
}
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
function loadManifest(name: string): { manifest: string; broker: boolean } {
@@ -204,7 +191,7 @@ function loadManifest(name: string): { manifest: string; broker: boolean } {
const remap = REMAP[name] ?? {};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image));
if (typeof r.image === "string") r.image = pinned(r.image);
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
}
const manifest = JSON.stringify(m);
@@ -259,7 +246,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
test/integration/whole-mesh-full.test.ts
+686 -199
View File
File diff suppressed because it is too large Load Diff
test/integration/whole-mesh-novox.test.ts
+12 -26
View File
@@ -42,7 +42,8 @@ import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { labIsUsable, destroyAll, substrateBundle, onTheMachine } from "./harness.ts";
import type { HeldImage } from "../../src/pinning.ts";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -160,8 +161,8 @@ const REMAP: Record<string, Record<string, string>> = {
};
let instanceId = "";
/** What the scenario's registry serves, by digest. */
let stocked: string[] = [];
/** The mesh's own images, as the machines hold them. */
let held: HeldImage[] = [];
function quote(s: string): string {
return `'${s.replaceAll("'", `'\\''`)}'`;
@@ -187,30 +188,15 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}
/** The repository path a reference serves under — registry.ts's repositoryFor, mirrored. */
function repositoryFor(reference: string): string {
const withoutDigest = reference.split("@")[0] ?? reference;
const lastColon = withoutDigest.lastIndexOf(":");
const lastSlash = withoutDigest.lastIndexOf("/");
return lastColon > lastSlash ? withoutDigest.slice(0, lastColon) : withoutDigest;
}
/** The pinned reference this scenario's registry serves for a repository. */
function pinned(repository: string): string {
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
assert.ok(found, `the scenario stocks no ${repository}; it serves\n ${stocked.join("\n ")}`);
return found;
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
function pinned(reference: string): string {
return onTheMachine(reference, held);
}
/** The substrate bundle, its image references pointed at this scenario's own registry. */
function bundleFor(images: string[]): string {
let text = readFileSync(bundle, "utf8");
for (const ref of images) {
const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
}
return text;
/** The substrate bundle: ours by the ID the machine holds, everything else upstream. */
function bundleFor(images: HeldImage[]): string {
return substrateBundle(bundle, images);
}
/**
@@ -226,7 +212,7 @@ function loadManifest(name: string): { manifest: string; broker: boolean } {
const remap = REMAP[name] ?? {};
for (const r of m.resources ?? []) {
if (r.type !== "container") continue;
if (typeof r.image === "string") r.image = pinned(repositoryFor(r.image));
if (typeof r.image === "string") r.image = pinned(r.image);
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
}
const manifest = JSON.stringify(m);
@@ -282,7 +268,7 @@ before(async () => {
onProgress: (m) => console.log(`raise: ${m}`),
});
instanceId = raised.instanceId;
stocked = raised.images;
held = raised.images;
// anchor raises the substrate from its bundle, digests rewritten to the scenario registry's.
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
test/invariants.test.ts
+2 -2
View File
@@ -23,8 +23,8 @@ test("the same address on different segments is NOT a conflict", () => {
// Every private network has its own `.1`. Reporting that would make the check useless.
assert.deepEqual(
duplicateAddresses([
{ machine: "gw-a", segment: "home", address: "192.168.1.1/24" },
{ machine: "gw-b", segment: "cafe", address: "192.168.1.1/24" },
{ machine: "gw-a", segment: "home", address: "10.99.1.1/24" },
{ machine: "gw-b", segment: "cafe", address: "10.99.1.1/24" },
]),
[],
);
test/pinning.test.ts
+129 -38
View File
@@ -1,73 +1,164 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { pinnedInto, repositoryOf, stillUnpinned } from "../src/pinning.ts";
import {
isMeshBuilt, mustBeHandedOver, pinnedInto, referenceFor, repositoryOf, stillUnpinned,
type HeldImage,
} from "../src/pinning.ts";
const SERVED = [
"192.0.2.250:5000/postgres@sha256:" + "a".repeat(64),
"192.0.2.250:5000/mesh-provision-postgres@sha256:" + "b".repeat(64),
"192.0.2.250:5000/gitea/gitea@sha256:" + "c".repeat(64),
"192.0.2.250:5000/ghcr.io/mailu/admin@sha256:" + "d".repeat(64),
/**
* What a machine holds, and what it does not.
*
* The lab used to raise a registry inside the scenario and rewrite EVERY reference to it —
* third-party ones included. That registry exists in no production mesh, so what these tests
* describe now is the real division: our images are handed over and named by their own ID,
* everything else is pulled from the internet and left exactly as written.
*/
const HELD: HeldImage[] = [
{
requested: "mesh-control:development",
repository: "mesh-control",
reference: "sha256:" + "a".repeat(64),
},
{
requested: "mesh-runtime-postgres:development",
repository: "mesh-runtime-postgres",
reference: "sha256:" + "b".repeat(64),
},
{
requested: "mesh-route-proxy:development",
repository: "mesh-route-proxy",
reference: "sha256:" + "c".repeat(64),
},
];
test("the repository is what survives being served somewhere else", () => {
assert.equal(repositoryOf(SERVED[0]!), "postgres");
assert.equal(repositoryOf(SERVED[2]!), "gitea/gitea");
assert.equal(repositoryOf(SERVED[3]!), "ghcr.io/mailu/admin");
test("the repository is the reference without its tag", () => {
assert.equal(repositoryOf("mesh-runtime-postgres:development"), "mesh-runtime-postgres");
assert.equal(repositoryOf("alpine"), "alpine");
assert.equal(repositoryOf("gitea/gitea:1.22"), "gitea/gitea");
assert.equal(repositoryOf("ghcr.io/mailu/admin@sha256:" + "d".repeat(64)), "ghcr.io/mailu/admin");
// A port in a hostname is a colon that is NOT a tag, and treating it as one would truncate the
// host rather than the tag.
assert.equal(
repositoryOf("registry.example:5000/novox/www:latest"), "registry.example:5000/novox/www");
});
/**
* The line the whole change turns on.
*
* An image with somewhere to be fetched from is fetched from there. An image with nowhere — no
* registry host, no upstream organisation, and a `mesh-` name — is one built here and handed over.
*/
test("only what is built here and published nowhere counts as ours", () => {
for (const ours of [
"mesh-control:development", "mesh-runtime-plex:development", "mesh-route-proxy:development",
"mesh-provision-postgres@sha256:" + "0".repeat(64),
]) {
assert.ok(isMeshBuilt(ours), `${ours} is one of ours and was not recognised`);
}
for (const theirs of [
"postgres:17-alpine", "gitea/gitea:1.22", "ghcr.io/mailu/admin:1.9",
"registry.example:5000/novox/www:latest",
// A registry host in front of one of our names does NOT make it ours: it says somebody
// published it, so the machine can fetch it from there like anything else.
"registry.example:5000/mesh-control:development",
]) {
assert.ok(!isMeshBuilt(theirs), `${theirs} is not ours and was claimed`);
}
});
// The case this exists for: an image the mesh builds has no digest until it is built, so a
// manifest ships sixty-four zeros and would stop on the machine (novox/hq 04-ISSUES/025).
test("a placeholder for one of our own images becomes the one this scenario serves", () => {
const before = `"image": "mesh-provision-postgres@sha256:${"0".repeat(64)}"`;
const after = pinnedInto(before, SERVED);
assert.match(after, /192\.0\.2\.250:5000\/mesh-provision-postgres@sha256:b{64}/);
test("a placeholder for one of our own images becomes the image the machine holds", () => {
const before = `"image": "mesh-runtime-postgres@sha256:${"0".repeat(64)}"`;
const after = pinnedInto(before, HELD);
assert.equal(after, `"image": "sha256:${"b".repeat(64)}"`);
assert.deepEqual(stillUnpinned(after), []);
});
// And a real third-party digest is replaced too — the text says which image, the scenario says
// which copy of it.
test("a real digest is redirected to this scenario's copy", () => {
const before = `"image": "gitea/gitea@sha256:${"f".repeat(64)}"`;
assert.match(pinnedInto(before, SERVED), /192\.0\.2\.250:5000\/gitea\/gitea@sha256:c{64}/);
/**
* **The heart of it.** A third-party reference is not touched.
*
* It used to be rewritten to whatever the lab's registry assigned, which meant the bed never once
* fetched an image the way a real machine does — and every bootstrap fault that depended on that
* went unfound.
*/
test("a third-party image is left exactly as the manifest wrote it", () => {
for (const reference of [
`postgres@sha256:${"7".repeat(64)}`,
`gitea/gitea@sha256:${"8".repeat(64)}`,
`ghcr.io/mailu/admin@sha256:${"9".repeat(64)}`,
`registry.example:5000/novox/www:latest`,
]) {
const before = `"image": "${reference}"`;
assert.equal(pinnedInto(before, HELD), before, `${reference} was rewritten`);
}
});
test("a reference that already carries a registry is still redirected", () => {
const before = `"image": "docker.io/postgres@sha256:${"e".repeat(64)}"`;
assert.match(pinnedInto(before, SERVED), /192\.0\.2\.250:5000\/postgres@sha256:a{64}/);
});
// **Left alone, not blanked.** A repository this scenario did not stock may be reachable some
// other way, and emptying the reference would produce the exact failure this prevents.
test("something the scenario does not serve is untouched", () => {
const before = `"image": "redis@sha256:${"9".repeat(64)}"`;
assert.equal(pinnedInto(before, SERVED), before);
test("a reference of ours that already carries a registry is still redirected", () => {
// What the committed substrate bundle looks like: written for a target that had a registry.
const before = `"image": "192.0.2.250:5000/mesh-control@sha256:${"e".repeat(64)}"`;
assert.equal(pinnedInto(before, HELD), `"image": "sha256:${"a".repeat(64)}"`);
});
// A longer repository ending in a shorter one must not be half-replaced.
test("a repository that ends in another one is not partly rewritten", () => {
const before = `"image": "my-postgres@sha256:${"7".repeat(64)}"`;
assert.equal(pinnedInto(before, SERVED), before,
"'my-postgres' was rewritten because it ends in 'postgres'");
const before = `"image": "our-mesh-control@sha256:${"7".repeat(64)}"`;
assert.equal(pinnedInto(before, HELD), before,
"'our-mesh-control' was rewritten because it ends in 'mesh-control'");
});
test("every image in a whole manifest is redirected at once", () => {
test("every image in a whole manifest is settled at once", () => {
const manifest = JSON.stringify({
resources: [
{ id: "db", image: `postgres@sha256:${"1".repeat(64)}` },
{ id: "prov", image: `mesh-provision-postgres@sha256:${"0".repeat(64)}` },
{ id: "runtime", image: `mesh-runtime-postgres@sha256:${"0".repeat(64)}` },
{ id: "proxy", image: `mesh-route-proxy@sha256:${"0".repeat(64)}` },
{ id: "app", image: `gitea/gitea@sha256:${"2".repeat(64)}` },
],
});
const after = pinnedInto(manifest, SERVED);
const after = pinnedInto(manifest, HELD);
assert.deepEqual(stillUnpinned(after), []);
for (const want of ["a".repeat(64), "b".repeat(64), "c".repeat(64)]) {
assert.ok(after.includes(want), `missing ${want.slice(0, 6)}… in ${after}`);
}
assert.ok(after.includes(`sha256:${"b".repeat(64)}`), after);
assert.ok(after.includes(`sha256:${"c".repeat(64)}`), after);
// And the two that are not ours are still whole, digest and all.
assert.ok(after.includes(`postgres@sha256:${"1".repeat(64)}`), after);
assert.ok(after.includes(`gitea/gitea@sha256:${"2".repeat(64)}`), after);
});
test("what a repository is held under can be asked for, and absence is not an empty string", () => {
assert.equal(referenceFor(HELD, "mesh-control"), `sha256:${"a".repeat(64)}`);
assert.equal(referenceFor(HELD, "mesh-runtime-plex"), undefined);
});
test("what is still a placeholder can be named", () => {
const text = `"image": "something-of-ours@sha256:${"0".repeat(64)}"`;
assert.deepEqual(stillUnpinned(text), ["something-of-ours"]);
});
/**
* An image a machine cannot fetch by itself has to be handed to it, and there are two ways to be in
* that position: built here and published nowhere, or sitting in a registry the machine has no
* account for. The second was found by deleting the lab's registry — the operator's own images
* failed with `no basic auth credentials`, which no amount of retrying improves.
*/
test("an image the machine cannot fetch by itself is handed over", () => {
for (const handed of [
"mesh-control:development",
"mesh-runtime-plex:development",
`registry.example/novox/www@sha256:${"a".repeat(64)}`,
"registry.example:5000/novox/photos-server:latest",
]) {
assert.ok(mustBeHandedOver(handed), `${handed} cannot be fetched and was not handed over`);
}
for (const fetched of [
"postgres:17-alpine",
"gitea/gitea:1.22",
"ghcr.io/mailu/admin:1.9",
"quay.io/keycloak/keycloak:26",
"lscr.io/linuxserver/sonarr:latest",
"mcr.microsoft.com/mssql/server:2022-latest",
]) {
assert.ok(!mustBeHandedOver(fetched), `${fetched} can be fetched and was handed over anyway`);
}
});
test/place.test.ts
+72 -1
View File
@@ -1,7 +1,7 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { parseScenario } from "../src/declaration/parse.ts";
import { planPlacements, PLACEABLE, isPlaceable } from "../src/lifecycle/place.ts";
import { planPlacements, planHeldImages, PLACEABLE, isPlaceable } from "../src/lifecycle/place.ts";
import { assertSupported, UnsupportedError } from "../src/lifecycle/supported.ts";
/**
@@ -59,6 +59,77 @@ test("placing the host is supported", () => {
assert.doesNotThrow(() => assertSupported(scenario("place:\n all: [host]")));
});
/**
* Which machine is handed which of the mesh's own images.
*
* **Not an economy — a fact.** An operator's workstation holds the images its own modules need,
* because somebody put them there, and a home server holds a different set. The lab used to serve
* everything to everyone from a registry it raised, which hid that entirely; handing every machine
* the union instead would put some thirty gigabytes of runtimes onto whole-mesh-full's
* thirty-gigabyte workstations, and the raise would die on disk with the topology looking fine.
*/
test("a machine that says nothing is handed everything the scenario has", () => {
const s = parseScenario(`
scenario: s
segments: { hosting: { kind: public, cidr: [192.0.2.0/24] } }
machines:
anchor: { at: { segment: hosting, address: [192.0.2.10] }, egress: true }
images: [mesh-control:development, mesh-runtime-redis:development]
place: { all: [host, runtime] }
`);
assert.deepEqual(planHeldImages(s), [
{ machine: "anchor", images: ["mesh-control:development", "mesh-runtime-redis:development"] },
]);
});
test("a machine that names some is handed those, and no others", () => {
const s = parseScenario(`
scenario: s
segments: { hosting: { kind: public, cidr: [192.0.2.0/24] } }
machines:
anchor:
at: { segment: hosting, address: [192.0.2.10] }
egress: true
images: [mesh-control:development]
laptop:
at: { segment: hosting, address: [192.0.2.20] }
egress: true
images: [mesh-runtime-redis:development]
images: [mesh-control:development, mesh-runtime-redis:development]
place: { all: [host, runtime] }
`);
assert.deepEqual(planHeldImages(s), [
{ machine: "anchor", images: ["mesh-control:development"] },
{ machine: "laptop", images: ["mesh-runtime-redis:development"] },
]);
});
test("a machine that names none is handed none, and is not a machine to visit", () => {
// Absent and empty are different, and a machine running nothing of ours should be able to say
// so without the lab deciding it must have meant everything.
const s = parseScenario(`
scenario: s
segments: { hosting: { kind: public, cidr: [192.0.2.0/24] } }
machines:
anchor: { at: { segment: hosting, address: [192.0.2.10] }, egress: true }
bare: { at: { segment: hosting, address: [192.0.2.20] }, egress: true, images: [] }
images: [mesh-control:development]
place: { all: [host, runtime] }
`);
assert.deepEqual(planHeldImages(s).map((p) => p.machine), ["anchor"]);
});
test("a scenario with none of our images loads nothing anywhere", () => {
const s = parseScenario(`
scenario: s
segments: { hosting: { kind: public, cidr: [192.0.2.0/24] } }
machines:
anchor: { at: { segment: hosting, address: [192.0.2.10] } }
place: { all: [host] }
`);
assert.deepEqual(planHeldImages(s), []);
});
test("a tier that does not exist is refused BY NAME", () => {
// Named individually rather than refused as a whole, so a scenario placing a host and a
// substrate is told exactly which half the lab cannot do — rather than being told `place:`
test/rebuild.test.ts
+48 -1
View File
@@ -1,7 +1,8 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { planned } from "../src/rebuild.ts";
import { planned, controlPlaneImage } from "../src/rebuild.ts";
import { repositories } from "../src/repos.ts";
import { loadScenario } from "../src/declaration/parse.ts";
// The control plane's image and the builder are one step, not two.
//
@@ -39,6 +40,52 @@ test("every image the lab runs is rebuilt, not only the control plane's", () =>
}
});
// The installer is built, and it is built AFTER the image it carries.
//
// `make bootstrap` embeds the output of `docker save <image>`, so an installer built before the
// control plane's image is one carrying whatever was lying around — 04-ISSUES/005 again, this time
// sealed inside a binary where nothing would ever notice. The bed raises its anchor by running this
// program (novox/hq ADR 0067), so a stale one is a bed proving something about last week.
test("the installer is built, carrying the image built in the same run", () => {
const builds = planned({
MESH_LAB_HOST_BINARY: "/repo/host/mesh-host",
MESH_LAB_MODULES: "/repo/control/examples/modules",
MESH_LAB_BOOTSTRAP_BINARY: "/repo/host/mesh-bootstrap",
});
const what = builds.map((b) => b.what);
assert.ok(what.includes("installer"), "the installer is never built, so the bed carries a stale one");
assert.ok(
what.indexOf("images") < what.indexOf("installer"),
`the installer is built before the image it embeds: ${what.join(", ")}`,
);
const installer = builds.find((b) => b.what === "installer")!;
assert.equal(installer.in, "/repo/host");
assert.ok(installer.argv.includes(`IMAGE=${controlPlaneImage({})}`), installer.argv.join(" "));
assert.ok(installer.argv.includes("BOOTSTRAP_OUT=/repo/host/mesh-bootstrap"), installer.argv.join(" "));
});
// The anchor must NOT be handed the control plane's image.
//
// The installer carries it, which is the whole reason a machine that can reach no registry can
// raise a mesh (novox/hq ADR 0067). Hand it over from the workstation as well and the installer's
// load says "already held", the carrying is never exercised, and the bed goes green on a fiction —
// the same class of thing the lab's own registry used to hide. Asserted on the file rather than
// remembered, because a list of images is exactly the kind of thing somebody tops up.
test("the whole-mesh bed hands its anchor no control-plane image", () => {
const scenario = loadScenario("scenarios/whole-mesh-full.yml");
const named = [
...(scenario.images ?? []),
...Object.values(scenario.machines).flatMap((m) => m.images ?? []),
];
assert.deepEqual(
named.filter((i) => i.startsWith("mesh-control")),
[],
"the anchor is handed mesh-control, so genesis would never find out whether the installer " +
"really carries it",
);
});
// A repository this run was not pointed at is not built, and not claimed.
test("only what this run was pointed at is built", () => {
assert.deepEqual(planned({}), []);
test/registry.test.ts
-66
View File
@@ -1,66 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { digestFrom, pinnedReference, registryAddress, repositoryFor } from "../src/lifecycle/registry.ts";
/**
* The registry inside a scenario (novox/hq 04-ISSUES/009).
*
* These test the pure parts. The parts that need a registry are exercised by raising a
* scenario, because a fake registry would assert that the fake behaves as expected
* (novox/hq ADR 0017).
*/
test("a digest is read from what the registry actually said", () => {
// The real shape of `docker push` output. The digest here is the REGISTRY's, not Docker
// Hub's, and that is the point: a declaration pins what this registry serves.
const output =
"The push refers to repository [localhost:5000/alpine]\n" +
"63f227048c13: Pushed\n" +
"3.20: digest: sha256:6c2a9711b0a9f32b0239d9222eb1072309cf46c6431d319ae249186d811a987c size: 528\n";
assert.equal(
digestFrom(output),
"sha256:6c2a9711b0a9f32b0239d9222eb1072309cf46c6431d319ae249186d811a987c",
);
});
test("no digest is not an empty digest", () => {
// A push that reported no digest leaves nothing for a declaration to pin, and inventing one
// would be worse than failing — the host would refuse it later, further from the cause.
assert.equal(digestFrom("The push refers to repository [localhost:5000/alpine]\n"), null);
assert.equal(digestFrom(""), null);
// Hex, but the wrong LENGTH. An earlier version used "tooshort", whose letters fall outside
// a-f — so it failed the character class and proved nothing about the length check.
assert.equal(digestFrom("digest: sha256:abc123"), null);
assert.equal(digestFrom("digest: sha256:" + "a".repeat(63)), null, "63 is not 64");
});
test("the repository is the reference without its tag", () => {
assert.equal(repositoryFor("alpine:3.20"), "alpine");
assert.equal(repositoryFor("alpine"), "alpine");
assert.equal(repositoryFor("library/postgres:17"), "library/postgres");
// A port in a hostname is a colon that is NOT a tag, and treating it as one would serve the
// image from a truncated path.
assert.equal(repositoryFor("localhost:5000/alpine:3.20"), "localhost:5000/alpine");
assert.equal(repositoryFor("localhost:5000/alpine"), "localhost:5000/alpine");
});
test("the registry's address is derived from its segment", () => {
assert.equal(registryAddress("192.0.2.0/24"), "192.0.2.250");
assert.equal(registryAddress("198.51.100.0/24"), "198.51.100.250");
// An IPv6-only segment cannot host it, and saying so beats producing an address nothing
// can be pointed at.
assert.throws(() => registryAddress("2001:db8:a::/48"), /not an IPv4 network/);
});
test("what a declaration pins is the registry's own digest", () => {
// Not Docker Hub's. ADR 0006 requires a reference that is exact and cannot move, and a
// digest this registry assigned is both.
const pinned = pinnedReference("192.0.2.250", {
requested: "alpine:3.20",
repository: "alpine",
digest: "sha256:" + "6".repeat(64),
});
assert.equal(pinned, `192.0.2.250:5000/alpine@sha256:${"6".repeat(64)}`);
assert.ok(pinned.includes("@sha256:"), "the host refuses anything not pinned by digest");
assert.ok(!pinned.includes(":3.20"), "a tag would move; the digest is what is pinned");
});
test/router.test.ts
+4 -4
View File
@@ -9,9 +9,9 @@ test("segments sharing a gateway declaration share ONE router", () => {
const scenario = parseScenario(`scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
iot: { kind: private, cidr: [192.168.30.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`);
machines: { a: { at: { segment: home, address: [10.99.1.9] } } }`);
const plans = planRouters(scenario, "inst");
assert.equal(plans.length, 1, "one gateway declaration, one router");
assert.deepEqual(plans[0]?.inside.sort(), ["home", "iot"]);
@@ -21,9 +21,9 @@ test("different external addresses mean different routers", () => {
const scenario = parseScenario(`scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
other: { kind: private, cidr: [192.168.30.0/24], gateway: { to: pub, address: [192.0.2.6], nat: [v4] } }
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`);
machines: { a: { at: { segment: home, address: [10.99.1.9] } } }`);
assert.equal(planRouters(scenario, "inst").length, 2);
});
test/supported.test.ts
+4 -4
View File
@@ -13,8 +13,8 @@ import { assertSupported, UnsupportedError } from "../src/lifecycle/supported.ts
const withGateway = `scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`;
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
machines: { a: { at: { segment: home, address: [10.99.1.9] } } }`;
test("a plain scenario is raisable", () => {
const scenario = parseScenario(`scenario: x
@@ -31,12 +31,12 @@ test("published ports and policy are implemented", () => {
const scenario = parseScenario(`scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
iot: { kind: private, cidr: [192.168.30.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
policy: [{ from: iot, to: home, allow: false }]
machines:
a:
at: { segment: home, address: [192.168.1.9] }
at: { segment: home, address: [10.99.1.9] }
published: [{ port: 443, on: home }]`);
assert.doesNotThrow(() => assertSupported(scenario));
});
test/validate.test.ts
+68 -11
View File
@@ -1,5 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { readdirSync } from "node:fs";
import { parseScenario } from "../src/declaration/parse.ts";
import { loadScenario } from "../src/declaration/parse.ts";
import { planRouters } from "../src/lifecycle/router.ts";
@@ -13,16 +14,21 @@ function refuses(yaml: string, pattern: RegExp): void {
}
test("the shipped scenarios are valid", () => {
for (const file of ["scenarios/bootstrap-single.yml", "scenarios/the-ordinary-shape.yml"]) {
assert.doesNotThrow(() => loadScenario(file));
// **Every one of them**, not a chosen two. Thirty-odd scenarios were rewritten in one pass when
// the lab's registry was removed, and a scenario nobody loads is a scenario nobody validates —
// which is how a bed goes unraisable for weeks and is only found when somebody wants it.
const files = readdirSync("scenarios").filter((f) => f.endsWith(".yml"));
assert.ok(files.length > 20, `only ${files.length} scenarios found — is the path right?`);
for (const file of files) {
assert.doesNotThrow(() => loadScenario(`scenarios/${file}`), `scenarios/${file}`);
}
});
test("a public segment on a private range is refused — the mesh would silently never form", () => {
refuses(
`scenario: x
segments: { net: { kind: public, cidr: [192.168.1.0/24] } }
machines: { a: { at: { segment: net, address: [192.168.1.1] } } }`,
segments: { net: { kind: public, cidr: [10.99.1.0/24] } }
machines: { a: { at: { segment: net, address: [10.99.1.1] } } }`,
/not documentation space/,
);
});
@@ -65,8 +71,8 @@ test("a gateway address must be on the PARENT segment, not the one behind it", (
`scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.168.1.1], nat: [v4] } }
machines: { a: { at: { segment: home, address: [192.168.1.9] } } }`,
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [10.99.1.1], nat: [v4] } }
machines: { a: { at: { segment: home, address: [10.99.1.9] } } }`,
/is not within 'pub'/,
);
});
@@ -114,7 +120,7 @@ test("publishing on a segment the machine is not attached to is refused", () =>
`scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
machines:
a:
at: { segment: pub, address: [192.0.2.10] }
@@ -170,12 +176,12 @@ test("a multi-homed machine is valid", () => {
parseScenario(`scenario: x
segments:
pub: { kind: public, cidr: [192.0.2.0/24] }
home: { kind: private, cidr: [192.168.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
home: { kind: private, cidr: [10.99.1.0/24], gateway: { to: pub, address: [192.0.2.5], nat: [v4] } }
machines:
border:
at:
- { segment: pub, address: [192.0.2.60] }
- { segment: home, address: [192.168.1.2] }`),
- { segment: home, address: [10.99.1.2] }`),
);
});
@@ -200,7 +206,7 @@ segments:
cidr: [198.51.100.0/24, "2001:db8:b::/48"]
home:
kind: private
cidr: [192.168.1.0/24]
cidr: [10.99.1.0/24]
gateway: { to: isp, address: [198.51.100.7, "2001:db8:b::7"], nat: [v4], forwardable: true, mapping_ttl: 120s }
devices:
kind: private
@@ -230,7 +236,7 @@ segments:
cidr: [198.51.100.0/24]
home:
kind: private
cidr: [192.168.1.0/24]
cidr: [10.99.1.0/24]
gateway: { to: isp, address: [198.51.100.7], nat: [v4], forwardable: true }
devices:
kind: private
@@ -251,6 +257,57 @@ machines: { a: { at: detached, egress: true } }`,
/detached but declares egress/);
});
/**
* `images:` is the mesh's own images and nothing else.
*
* **The rule that replaced the lab's registry.** Anything with somewhere to be fetched from is
* fetched from there, by the machine, over its uplink. Serving it from inside the scenario instead
* is what hid the bootstrap faults this lab exists to find — so it is refused rather than quietly
* done, or the fiction comes back one convenient line at a time.
*/
test("a third-party image in images: is refused, because nothing loads it", () => {
for (const image of ["postgres:17-alpine", "gitea/gitea:1.22", "ghcr.io/mailu/admin:1.9"]) {
refuses(`scenario: x
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
machines: { a: { at: { segment: net, address: [192.0.2.1] }, egress: true } }
images: ["${image}"]
place: { all: [runtime] }`,
/is not one of the mesh's own images/);
}
});
test("one of ours in images: is accepted", () => {
assert.doesNotThrow(() => parseScenario(`scenario: x
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
machines: { a: { at: { segment: net, address: [192.0.2.1] }, egress: true } }
images: [mesh-control:development, mesh-route-proxy:development]
place: { all: [runtime] }`));
});
test("images: is named by tag — an image ID is not knowable until the image is built", () => {
refuses(`scenario: x
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
machines: { a: { at: { segment: net, address: [192.0.2.1] }, egress: true } }
images: ["mesh-control@sha256:${"0".repeat(64)}"]
place: { all: [runtime] }`,
/is pinned by digest/);
});
test("a machine cannot be handed an image the scenario does not have", () => {
// Ignoring it silently would be a machine missing a runtime, failing several minutes later
// inside an apply, as a container that will not start.
refuses(`scenario: x
segments: { net: { kind: public, cidr: [192.0.2.0/24] } }
machines:
a:
at: { segment: net, address: [192.0.2.1] }
egress: true
images: [mesh-runtime-redis:development]
images: [mesh-control:development]
place: { all: [runtime] }`,
/is not in this scenario's images/);
});
test("a segment may not be named 'uplink' — the lab claims that name for egress", () => {
refuses(`scenario: x
segments: { uplink: { kind: public, cidr: [192.0.2.0/24] } }