Compare commits
67
Commits
cf6fcdff3c
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f94ee2dd0e | ||
|
|
62a02d7e94 | ||
|
|
08e3aa04e7 | ||
|
|
d93dc2628f | ||
|
|
23ca9cadad | ||
|
|
204a226e36 | ||
|
|
8d9e4bdb77 | ||
|
|
3f224c2876 | ||
|
|
a53dc8c586 | ||
|
|
bb746505dc | ||
|
|
514f7a46cd | ||
|
|
c9fd6d7887 | ||
|
|
4d2ec6e6a8 | ||
|
|
dba95f7e27 | ||
|
|
3ce66e8369 | ||
|
|
3f71b91fb8 | ||
|
|
2dfffd6dac | ||
|
|
36f2fd1c2c | ||
|
|
7a5d670e4e | ||
|
|
8a85e2d02e | ||
|
|
146d7da9ef | ||
|
|
10cfbd094a | ||
|
|
353cf88a4b | ||
|
|
9eaa3a623d | ||
|
|
a964ec287d | ||
|
|
2da442729f | ||
|
|
03bbc37572 | ||
|
|
484fafe799 | ||
|
|
7c81902571 | ||
|
|
080150addb | ||
|
|
fd1e5499d0 | ||
|
|
0d8eac88c3 | ||
|
|
2e0f11dfc2 | ||
|
|
e1739b1caf | ||
|
|
b0eddd28d8 | ||
|
|
2ce130c256 | ||
|
|
e64d296c80 | ||
|
|
acb8d3da88 | ||
|
|
f785f5892a | ||
|
|
d95174da02 | ||
|
|
2530ca762e | ||
|
|
107257faf4 | ||
|
|
690596d33b | ||
|
|
31163865d2 | ||
|
|
69d3813ae1 | ||
|
|
2bc1230252 | ||
|
|
b6be7ac8af | ||
|
|
38672cd356 | ||
|
|
8a3e7dbd1b | ||
|
|
ab5b0d11da | ||
|
|
26177d81be | ||
|
|
543ccb7380 | ||
|
|
85dc827660 | ||
|
|
2f11a29c4d | ||
|
|
64c081d025 | ||
|
|
467416728e | ||
|
|
e4c924a85e | ||
|
|
1b2443690d | ||
|
|
d7959001dd | ||
|
|
be58bd96f6 | ||
|
|
6083b9310a | ||
|
|
c8c3028f38 | ||
|
|
ec23e9f4cd | ||
|
|
8c37328ba3 | ||
|
|
e596758db9 | ||
|
|
2456b2f533 | ||
|
|
f2d29491b2 |
@@ -1 +1,2 @@
|
|||||||
node_modules/
|
node_modules/
|
||||||
|
node_modules
|
||||||
|
|||||||
@@ -165,7 +165,15 @@ export MESH_LAB_ROUTE_PROXY=<somewhere>/route-proxy
|
|||||||
|
|
||||||
`MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
|
`MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what
|
||||||
`suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
|
`suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and
|
||||||
claimed; leave it out and it is neither.
|
claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built as a
|
||||||
|
repository: a bed installs a catalogue module by reading its manifest from that checkout when it
|
||||||
|
runs, so the receipt names the catalogue's commit too, and a run taken before a manifest changed
|
||||||
|
says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds'
|
||||||
|
scenarios stock (`mesh-runtime-<module>:development`): each is compared against the module's
|
||||||
|
source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts
|
||||||
|
beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh`
|
||||||
|
where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075).
|
||||||
|
`--no-build` skips this too, and then the bed runs whatever image the store holds.
|
||||||
|
|
||||||
Check before running a long suite — it says which of these are missing rather than skipping
|
Check before running a long suite — it says which of these are missing rather than skipping
|
||||||
quietly:
|
quietly:
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# A MACHINE IN USE, ADOPTED — and then converged, and returned (novox/hq ADR 0100, ADR 0101).
|
||||||
|
#
|
||||||
|
# The mesh replaces a predecessor that is running on the same machines. The anchor here is
|
||||||
|
# prepared the way the predecessor leaves one: its own firewall (ufw) allowing a served port and
|
||||||
|
# denying the rest, a service container on that port under a name a catalogue module also uses, a
|
||||||
|
# file at a path that module declares, a stand-in for the predecessor's configuration sync that
|
||||||
|
# rewrites the file, and a container holding the registry's port. The bed then raises the mesh on
|
||||||
|
# it, adopted, and walks the migration the record decides.
|
||||||
|
#
|
||||||
|
# hosting (public)
|
||||||
|
# anchor 192.0.2.10 the machine in use: the predecessor, then the mesh adopted on it
|
||||||
|
# joiner 192.0.2.20 a fresh machine: the "second machine" that reaches the service and
|
||||||
|
# enrols through the found firewall; also where a converged genesis on a
|
||||||
|
# FRESH machine is asked (ADR 0101)
|
||||||
|
# outsider 192.0.2.30 never enrolled, never on the private network: the probe from outside,
|
||||||
|
# and the lab's forge — the bed serves the checkouts under test to the
|
||||||
|
# anchor's builder from here, so nothing on the workstation listens
|
||||||
|
#
|
||||||
|
# inbound: allow on every machine — the anchor's firewall is the predecessor's, installed by the
|
||||||
|
# bed; `inbound: deny` would load the lab's own table beside it and make that the thing under test.
|
||||||
|
scenario: adoption
|
||||||
|
|
||||||
|
segments:
|
||||||
|
hosting:
|
||||||
|
kind: public
|
||||||
|
cidr: [192.0.2.0/24]
|
||||||
|
|
||||||
|
machines:
|
||||||
|
# Sized like the one-node bed's anchor: genesis builds the control plane, the base and the
|
||||||
|
# catalogue's modules here, beside the predecessor's two containers.
|
||||||
|
anchor:
|
||||||
|
at: { segment: hosting, address: [192.0.2.10] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 12GiB
|
||||||
|
cpus: 6
|
||||||
|
disk: 60GiB
|
||||||
|
|
||||||
|
joiner:
|
||||||
|
at: { segment: hosting, address: [192.0.2.20] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 3GiB
|
||||||
|
cpus: 2
|
||||||
|
disk: 20GiB
|
||||||
|
|
||||||
|
outsider:
|
||||||
|
at: { segment: hosting, address: [192.0.2.30] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 2GiB
|
||||||
|
cpus: 2
|
||||||
|
disk: 15GiB
|
||||||
|
|
||||||
|
place:
|
||||||
|
all: [host, runtime]
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
# One machine that becomes a mesh and grants a consumer an S3 bucket from an assigned minio provider.
|
|
||||||
#
|
|
||||||
# The postgres bed proves the provider/consumer contract for a database; this proves it for object
|
|
||||||
# storage (novox/hq ADR 0052/0053), on a provider whose code drives the `mc` CLI (so the runtime image
|
|
||||||
# carries it): minio is assigned, a consumer that requires s3-bucket is assigned, and the mesh mints
|
|
||||||
# one secret key; minio's provisioner creates a bucket and a service account under the access key the
|
|
||||||
# mesh derived with the secret it minted, and the consumer reaches its bucket with only that.
|
|
||||||
scenario: minio-node
|
|
||||||
|
|
||||||
segments:
|
|
||||||
hosting:
|
|
||||||
kind: public
|
|
||||||
cidr: [192.0.2.0/24]
|
|
||||||
|
|
||||||
machines:
|
|
||||||
anchor:
|
|
||||||
at: { segment: hosting, address: [192.0.2.10] }
|
|
||||||
egress: true
|
|
||||||
inbound: allow
|
|
||||||
memory: 3GiB
|
|
||||||
cpus: 2
|
|
||||||
|
|
||||||
images:
|
|
||||||
- mesh-controller:development
|
|
||||||
# minio's runtime, built by scripts/build-module-runtime.sh minio (it carries mc), loaded onto
|
|
||||||
# the machine.
|
|
||||||
- mesh-runtime-minio:development
|
|
||||||
|
|
||||||
place:
|
|
||||||
all: [host, runtime]
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
# One machine that becomes a mesh and then assigns itself plex's tool runtime.
|
|
||||||
#
|
|
||||||
# The audit-node bed proved an assigned *consumer* (novox/hq ADR 0048). This proves an assigned
|
|
||||||
# module that *serves tools* (ADR 0052): the same first-node foundation, plus plex's tool runtime on
|
|
||||||
# top. The node enrols itself, the mesh issues plex a broker account scoped to serve.plex.* and
|
|
||||||
# assigns it, the host runs the runtime container, and a caller invokes plex.plex_reachable over the
|
|
||||||
# mesh — proof the module runs its own code as its own process under its own scoped account.
|
|
||||||
scenario: plex-node
|
|
||||||
|
|
||||||
segments:
|
|
||||||
hosting:
|
|
||||||
kind: public
|
|
||||||
cidr: [192.0.2.0/24]
|
|
||||||
|
|
||||||
machines:
|
|
||||||
anchor:
|
|
||||||
at: { segment: hosting, address: [192.0.2.10] }
|
|
||||||
egress: true
|
|
||||||
inbound: allow
|
|
||||||
memory: 3GiB
|
|
||||||
cpus: 2
|
|
||||||
|
|
||||||
images:
|
|
||||||
- mesh-controller:development
|
|
||||||
# Plex's tool runtime, built by scripts/build-module-runtime.sh plex into the local daemon and
|
|
||||||
# loaded onto the machine, which holds it by its own image ID.
|
|
||||||
- mesh-runtime-plex:development
|
|
||||||
|
|
||||||
place:
|
|
||||||
all: [host, runtime]
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# One machine that becomes a mesh and grants a consumer a database from an assigned postgres provider.
|
|
||||||
#
|
|
||||||
# The redis mesh-grant bed proves the whole provider/consumer contract for a cache; this proves it for
|
|
||||||
# a database (novox/hq ADR 0052/0053): postgres's runtime carries psql, its provisioner creates a role
|
|
||||||
# and database under the login the mesh derived with the password the mesh minted, and a consumer
|
|
||||||
# connects to its own database with only what the mesh delivered.
|
|
||||||
scenario: postgres-node
|
|
||||||
|
|
||||||
segments:
|
|
||||||
hosting:
|
|
||||||
kind: public
|
|
||||||
cidr: [192.0.2.0/24]
|
|
||||||
|
|
||||||
machines:
|
|
||||||
anchor:
|
|
||||||
at: { segment: hosting, address: [192.0.2.10] }
|
|
||||||
egress: true
|
|
||||||
inbound: allow
|
|
||||||
memory: 3GiB
|
|
||||||
cpus: 2
|
|
||||||
|
|
||||||
images:
|
|
||||||
- mesh-controller:development
|
|
||||||
# postgres's runtime, built by scripts/build-module-runtime.sh postgres (it carries psql), loaded
|
|
||||||
# onto the machine.
|
|
||||||
- mesh-runtime-postgres:development
|
|
||||||
|
|
||||||
place:
|
|
||||||
all: [host, runtime]
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# One machine that becomes a mesh and then assigns itself redis — a *provider* module.
|
# One machine that becomes a mesh and then assigns itself redis — a *provider* module.
|
||||||
#
|
#
|
||||||
# plex-node proves an assigned module that serves tools (novox/hq ADR 0052). This proves the same
|
# A bed proving an assigned module that serves tools (novox/hq ADR 0052) once lived beside this; this proves the same
|
||||||
# for a provider: redis's runtime runs its provisioner AND its tools as one process under one scoped
|
# for a provider: redis's runtime runs its provisioner AND its tools as one process under one scoped
|
||||||
# broker account. The provisioner emitting a lifecycle event is the thing 0052 fixes — before it,
|
# broker account. The provisioner emitting a lifecycle event is the thing 0052 fixes — before it,
|
||||||
# the provisioner ran in a container with no broker and its emit could not fire.
|
# the provisioner ran in a container with no broker and its emit could not fire.
|
||||||
@@ -24,6 +24,9 @@ images:
|
|||||||
# Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local
|
# Redis's tool+provisioner runtime, built by scripts/build-module-runtime.sh redis into the local
|
||||||
# daemon and loaded onto the machine, which holds it by its own image ID.
|
# daemon and loaded onto the machine, which holds it by its own image ID.
|
||||||
- mesh-runtime-redis:development
|
- mesh-runtime-redis:development
|
||||||
|
# The vault's, for the beds that install the catalogue's redis: its own password is a secret the
|
||||||
|
# vault provides (novox/hq ADR 0085).
|
||||||
|
- mesh-runtime-mesh-vault:development
|
||||||
|
|
||||||
place:
|
place:
|
||||||
all: [host, runtime]
|
all: [host, runtime]
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
# One machine that becomes a mesh and assigns itself sonarr's tool runtime.
|
|
||||||
#
|
|
||||||
# plex-node proved a tools+events module that self-detects its token from a mounted config dir; this
|
|
||||||
# proves the same self-configuring pattern generalises to the Servarr family (novox/hq ADR 0052):
|
|
||||||
# sonarr's runtime detects its API key from the server's config.xml and serves sonarr's tools over a
|
|
||||||
# mesh-issued scoped account, with no live Sonarr to reach.
|
|
||||||
scenario: sonarr-node
|
|
||||||
|
|
||||||
segments:
|
|
||||||
hosting:
|
|
||||||
kind: public
|
|
||||||
cidr: [192.0.2.0/24]
|
|
||||||
|
|
||||||
machines:
|
|
||||||
anchor:
|
|
||||||
at: { segment: hosting, address: [192.0.2.10] }
|
|
||||||
egress: true
|
|
||||||
inbound: allow
|
|
||||||
memory: 3GiB
|
|
||||||
cpus: 2
|
|
||||||
|
|
||||||
images:
|
|
||||||
- mesh-controller:development
|
|
||||||
- mesh-runtime-sonarr:development
|
|
||||||
|
|
||||||
place:
|
|
||||||
all: [host, runtime]
|
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# The control plane's store restarting while a machine joins (novox/hq issue 083).
|
||||||
|
#
|
||||||
|
# Adopting the foundation's store — the first thing a control-node does, and the first thing a
|
||||||
|
# migration does — recreates it, and for those seconds the control plane cannot write. A machine
|
||||||
|
# enrolling then used to be refused, or worse, left with its token spent and no identity. This
|
||||||
|
# raises the foundation on `anchor`, takes its store away, has `laptop` enrol into the gap and
|
||||||
|
# brings the store back: the enrolment must complete on its own.
|
||||||
|
scenario: store-window
|
||||||
|
|
||||||
|
segments:
|
||||||
|
hosting:
|
||||||
|
kind: public
|
||||||
|
cidr: [192.0.2.0/24]
|
||||||
|
|
||||||
|
machines:
|
||||||
|
anchor:
|
||||||
|
at: { segment: hosting, address: [192.0.2.10] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 3GiB
|
||||||
|
cpus: 2
|
||||||
|
laptop:
|
||||||
|
at: { segment: hosting, address: [192.0.2.20] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 1GiB
|
||||||
|
|
||||||
|
images:
|
||||||
|
- mesh-controller:development
|
||||||
|
|
||||||
|
place:
|
||||||
|
all: [host, runtime]
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# One machine that becomes a mesh, runs the mesh's own certificate authority, and is then given the
|
||||||
|
# module that makes it trust it — the bed for novox/hq ADR 0147 and issue 129.
|
||||||
|
#
|
||||||
|
# The question is narrow and the bed is shaped to answer only it: does a machine holding `ca-trust`
|
||||||
|
# verify a certificate from the mesh's own authority with no bundle argument and no `-k`, and does
|
||||||
|
# it stop verifying it when the module is taken away? The authority itself is what is dialled —
|
||||||
|
# step-ca serves its own API with a leaf it issued — so nothing else has to be right for the answer
|
||||||
|
# to mean something. No proxy, no routed name, no public issuance: those are the certificates and
|
||||||
|
# route-forwarding beds, and a trust bed that leaned on them would pass for their reasons.
|
||||||
|
#
|
||||||
|
# The negative half is not optional. It is asserted BEFORE the module is assigned and again AFTER it
|
||||||
|
# is unassigned, because an anchor bed that only ever checks the success is one that would pass on a
|
||||||
|
# machine that already trusted everything.
|
||||||
|
#
|
||||||
|
# MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||||
|
# MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||||
|
# step-ca's image is upstream and pinned by the catalogue; the machine pulls it over its uplink.
|
||||||
|
# ca-trust carries no image at all — a script, a unit, and the machine's own systemd.
|
||||||
|
scenario: trust-anchor
|
||||||
|
|
||||||
|
segments:
|
||||||
|
hosting:
|
||||||
|
kind: public
|
||||||
|
cidr: [192.0.2.0/24]
|
||||||
|
|
||||||
|
machines:
|
||||||
|
anchor:
|
||||||
|
at: { segment: hosting, address: [192.0.2.10] }
|
||||||
|
egress: true
|
||||||
|
inbound: allow
|
||||||
|
memory: 3GiB
|
||||||
|
cpus: 2
|
||||||
|
|
||||||
|
images:
|
||||||
|
- mesh-controller:development
|
||||||
|
|
||||||
|
place:
|
||||||
|
# Only the host. The authority's image comes from the internet over the machine's uplink, and the
|
||||||
|
# trust module has nothing to place.
|
||||||
|
all: [host]
|
||||||
@@ -3,11 +3,11 @@
|
|||||||
# The app-postgres provider and the mesh's own foundation store both want host port 5432, so they
|
# The app-postgres provider and the mesh's own foundation store both want host port 5432, so they
|
||||||
# cannot share a machine — the collision that blocked this chain single-node. Here the foundation
|
# cannot share a machine — the collision that blocked this chain single-node. Here the foundation
|
||||||
# (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain —
|
# (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain —
|
||||||
# postgres and redis PROVIDERS plus the baserow and letta CONSUMERS that require them. Both
|
# the baserow and letta CONSUMERS of the one store (baserow keeps its cache inside its own container,
|
||||||
|
# novox/hq 081). Both
|
||||||
# machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor,
|
# machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor,
|
||||||
# over the underlay both machines already share. Provider and consumers are co-located on laptop, so
|
# over the underlay both machines already share. The consumers' databases are minted on the one
|
||||||
# no cross-node module comms and no overlay are needed — and the 5432-vs-foundation conflict is gone
|
# foundation store on anchor and reached over the overlay; laptop runs no provider of its own.
|
||||||
# because the foundation store is on the OTHER node.
|
|
||||||
scenario: two-node-db
|
scenario: two-node-db
|
||||||
|
|
||||||
segments:
|
segments:
|
||||||
@@ -25,8 +25,7 @@ machines:
|
|||||||
inbound: allow
|
inbound: allow
|
||||||
memory: 4GiB
|
memory: 4GiB
|
||||||
cpus: 4
|
cpus: 4
|
||||||
# The whole DB-consumer chain: postgres + redis providers, each a server and a broker-bound
|
# The DB consumers: the baserow and letta services and their tools runtimes — a handful of
|
||||||
# runtime, plus the baserow and letta consumer services and their tools runtimes — a dozen
|
|
||||||
# containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server).
|
# containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server).
|
||||||
# At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says
|
# At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says
|
||||||
# so — and convergence would present as "the mesh hangs". Six gigabytes gives it room.
|
# so — and convergence would present as "the mesh hangs". Six gigabytes gives it room.
|
||||||
@@ -49,7 +48,6 @@ images:
|
|||||||
# provisioner (ADR 0048).
|
# provisioner (ADR 0048).
|
||||||
- mesh-runtime-postgres:development
|
- mesh-runtime-postgres:development
|
||||||
- mesh-runtime-lavinmq:development
|
- mesh-runtime-lavinmq:development
|
||||||
- mesh-runtime-redis:development
|
|
||||||
- mesh-runtime-baserow:development
|
- mesh-runtime-baserow:development
|
||||||
- mesh-runtime-letta:development
|
- mesh-runtime-letta:development
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set.
|
# scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set.
|
||||||
#
|
#
|
||||||
# Foundation (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the
|
# Foundation (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the
|
||||||
# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
|
# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres
|
||||||
# providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/
|
# providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/
|
||||||
# qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR
|
# qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR
|
||||||
# 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push —
|
# 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push —
|
||||||
|
|||||||
@@ -33,7 +33,16 @@ SRCS=(); for f in \
|
|||||||
pg.d.ts; do
|
pg.d.ts; do
|
||||||
[ -f "$MOD/$f" ] && SRCS+=("$f")
|
[ -f "$MOD/$f" ] && SRCS+=("$f")
|
||||||
done
|
done
|
||||||
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist >/dev/null )
|
# The module compiles against the SDK, which its package.json names and nothing installs: a module
|
||||||
|
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
|
||||||
|
# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
|
||||||
|
# the types; the image takes the SDK from MESH_SDK below.
|
||||||
|
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
|
||||||
|
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|
||||||
|
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
|
||||||
|
fi
|
||||||
|
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
|
||||||
|
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
|
||||||
|
|
||||||
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
||||||
cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
|
cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
|
||||||
|
|||||||
@@ -22,7 +22,18 @@ DOCKERFILE="$MESH_CATALOG/modules/route-proxy/Dockerfile"
|
|||||||
[ -f "$MESH_CONTROL/examples/route-proxy/main.go" ] || {
|
[ -f "$MESH_CONTROL/examples/route-proxy/main.go" ] || {
|
||||||
echo "no proxy source at $MESH_CONTROL/examples/route-proxy" >&2; exit 1; }
|
echo "no proxy source at $MESH_CONTROL/examples/route-proxy" >&2; exit 1; }
|
||||||
|
|
||||||
|
# The bases the manifest declares (novox/hq ADR 0097) are what this build starts FROM — the same
|
||||||
|
# images the mesh's builder would copy and hand the recipe, not the Dockerfile's floating defaults.
|
||||||
|
BASES=()
|
||||||
|
while IFS=$'\t' read -r arg image; do
|
||||||
|
[ -n "$arg" ] && BASES+=(--build-arg "$arg=$image")
|
||||||
|
done < <(python3 -c '
|
||||||
|
import json, sys
|
||||||
|
for on in json.load(open(sys.argv[1])).get("build", {}).get("on", []):
|
||||||
|
print(on["arg"], on["image"], sep="\t")
|
||||||
|
' "$MESH_CATALOG/modules/route-proxy/module.json")
|
||||||
|
|
||||||
# Context is the mesh-controller repository root: the proxy compiles against that module's go.mod and
|
# Context is the mesh-controller repository root: the proxy compiles against that module's go.mod and
|
||||||
# its examples/route-proxy package.
|
# its examples/route-proxy package.
|
||||||
docker build -f "$DOCKERFILE" -t "$TAG" "$MESH_CONTROL"
|
docker build -f "$DOCKERFILE" "${BASES[@]}" -t "$TAG" "$MESH_CONTROL"
|
||||||
echo "built $TAG (from $MESH_CONTROL/examples/route-proxy)"
|
echo "built $TAG (from $MESH_CONTROL/examples/route-proxy)"
|
||||||
|
|||||||
+11
-4
@@ -16,6 +16,7 @@
|
|||||||
|
|
||||||
import { spawnSync } from "node:child_process";
|
import { spawnSync } from "node:child_process";
|
||||||
import { repositories } from "./repos.ts";
|
import { repositories } from "./repos.ts";
|
||||||
|
import { plannedRuntimes } from "./runtimes.ts";
|
||||||
|
|
||||||
export interface Build {
|
export interface Build {
|
||||||
/** What it produces, for the log. */
|
/** What it produces, for the log. */
|
||||||
@@ -112,10 +113,16 @@ export function carriedImage(env: NodeJS.ProcessEnv = process.env): string {
|
|||||||
return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development";
|
return env["MESH_LAB_CARRIED_IMAGE"] ?? "mesh-builder:development";
|
||||||
}
|
}
|
||||||
|
|
||||||
/** rebuild runs the plan, and throws on the first failure rather than testing a stale artifact. */
|
/**
|
||||||
export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
|
* rebuild runs the plan, and throws on the first failure rather than testing a stale artifact.
|
||||||
|
*
|
||||||
|
* The plan is what the run was pointed at, plus the module runtimes the named beds stock where
|
||||||
|
* those are older than their source (novox/hq 04-ISSUES/075) — so a bed never again passes against
|
||||||
|
* a runtime built two weeks before the manifest it serves.
|
||||||
|
*/
|
||||||
|
export function rebuild(env: NodeJS.ProcessEnv = process.env, testFiles: string[] = []): string[] {
|
||||||
const built: string[] = [];
|
const built: string[] = [];
|
||||||
for (const build of planned(env)) {
|
for (const build of [...planned(env), ...plannedRuntimes(testFiles, env)]) {
|
||||||
const [command, ...args] = build.argv;
|
const [command, ...args] = build.argv;
|
||||||
const ran = spawnSync(command!, args, {
|
const ran = spawnSync(command!, args, {
|
||||||
cwd: build.in,
|
cwd: build.in,
|
||||||
@@ -127,7 +134,7 @@ export function rebuild(env: NodeJS.ProcessEnv = process.env): string[] {
|
|||||||
// the code in front of you, which is the whole of 005.
|
// the code in front of you, which is the whole of 005.
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`could not build the ${build.what}: ${build.argv.join(" ")} in ${build.in}\n\n` +
|
`could not build the ${build.what}: ${build.argv.join(" ")} in ${build.in}\n\n` +
|
||||||
`${(ran.stderr || ran.stdout || String(ran.error)).trim()}`,
|
`${(ran.stderr || ran.stdout || (ran.error ? String(ran.error) : `exit status ${ran.status}, and it said nothing`)).trim()}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
built.push(build.what);
|
built.push(build.what);
|
||||||
|
|||||||
+12
-1
@@ -10,7 +10,7 @@
|
|||||||
* pointed at is neither built nor claimed.
|
* pointed at is neither built nor claimed.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { dirname } from "node:path";
|
import { basename, dirname } from "node:path";
|
||||||
|
|
||||||
export interface Repositories {
|
export interface Repositories {
|
||||||
/** Absolute path to the repository root, by name. */
|
/** Absolute path to the repository root, by name. */
|
||||||
@@ -24,5 +24,16 @@ export function repositories(env: NodeJS.ProcessEnv = process.env): Repositories
|
|||||||
if (host) found["mesh-host"] = dirname(host);
|
if (host) found["mesh-host"] = dirname(host);
|
||||||
const modules = env["MESH_LAB_MODULES"];
|
const modules = env["MESH_LAB_MODULES"];
|
||||||
if (modules) found["mesh-controller"] = dirname(dirname(modules));
|
if (modules) found["mesh-controller"] = dirname(dirname(modules));
|
||||||
|
// The catalogue is read, not built: a bed installs a module by reading its manifest from this
|
||||||
|
// checkout at run time (novox/hq 04-ISSUES/073). A receipt that did not name the catalogue's
|
||||||
|
// commit could not say whether a catalogue change had been proven — the beds used to carry
|
||||||
|
// their own copies of the manifests, and then it could not.
|
||||||
|
const catalogue = env["MESH_LAB_CATALOG"];
|
||||||
|
if (catalogue) found["mesh-catalog"] = catalogueRoot(catalogue);
|
||||||
return found;
|
return found;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** MESH_LAB_CATALOG is accepted under either spelling — the checkout, or its `modules` directory. */
|
||||||
|
export function catalogueRoot(catalogue: string): string {
|
||||||
|
return basename(catalogue) === "modules" ? dirname(catalogue) : catalogue;
|
||||||
|
}
|
||||||
|
|||||||
+144
@@ -0,0 +1,144 @@
|
|||||||
|
/**
|
||||||
|
* The module runtimes a run stocks are rebuilt by the run, like everything else it tests.
|
||||||
|
*
|
||||||
|
* A per-module bed stocks the module's runtime image — the tool runtime carrying that module's
|
||||||
|
* code — from the workstation's image store, by tag. It was built by hand, by a script the suite
|
||||||
|
* never called, and on the day this was written the images for six modules about to run dated
|
||||||
|
* from two weeks before the manifests they were installed with (novox/hq 04-ISSUES/075). The
|
||||||
|
* suite's own rule, *the run rebuilds what it tests*, was held for the host and the control plane
|
||||||
|
* and not for these.
|
||||||
|
*
|
||||||
|
* So: for the beds about to run, every `mesh-runtime-<module>:development` their scenarios stock
|
||||||
|
* is compared against the source it is built from — the module in the catalogue, and the tool
|
||||||
|
* runtime and SDK it is built on — and rebuilt where the image is older, missing, or the source is
|
||||||
|
* uncommitted. A rebuild that fails stops the suite, the way a stale host binary would.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { parse } from "yaml";
|
||||||
|
import type { Build } from "./rebuild.ts";
|
||||||
|
import { catalogueRoot } from "./repos.ts";
|
||||||
|
|
||||||
|
/** A runtime image a scenario stocks by tag, and the module it is built from. */
|
||||||
|
export interface StockedRuntime {
|
||||||
|
tag: string;
|
||||||
|
module: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tags whose name is not the module's. The audit logger's runtime was the first, built before the
|
||||||
|
* script was generalised, and the scenario still stocks it under the module's slug.
|
||||||
|
*/
|
||||||
|
const NAMED_OTHERWISE: Record<string, string> = { "mesh-runtime-audit": "audit-logger" };
|
||||||
|
|
||||||
|
const RUNTIME_TAG = /^(mesh-runtime-[a-z0-9-]+):development$/;
|
||||||
|
|
||||||
|
/** The runtimes the scenarios of these beds stock, each named once. */
|
||||||
|
export function runtimesStockedBy(testFiles: string[], root: string = process.cwd()): StockedRuntime[] {
|
||||||
|
const seen = new Map<string, StockedRuntime>();
|
||||||
|
for (const file of testFiles) {
|
||||||
|
const text = readFileSync(resolve(root, file), "utf8");
|
||||||
|
const named = /const SCENARIO = "([^"]+)"/.exec(text);
|
||||||
|
if (!named) continue;
|
||||||
|
const scenario = parse(readFileSync(join(root, "scenarios", `${named[1]}.yml`), "utf8")) as { images?: unknown };
|
||||||
|
for (const image of Array.isArray(scenario.images) ? scenario.images : []) {
|
||||||
|
const m = RUNTIME_TAG.exec(String(image));
|
||||||
|
if (!m) continue;
|
||||||
|
const repository = m[1]!;
|
||||||
|
seen.set(repository, { tag: String(image), module: NAMED_OTHERWISE[repository] ?? repository.slice("mesh-runtime-".length) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [...seen.values()];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** When an image was made and when its source last changed, in seconds; null for no image. */
|
||||||
|
export interface Ages {
|
||||||
|
image: number | null;
|
||||||
|
/** Infinity where the source has uncommitted changes: what is on disk is newer than any commit. */
|
||||||
|
source: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** stale is whether the image predates its source, or is not there at all. */
|
||||||
|
export function isStale(a: Ages): boolean {
|
||||||
|
return a.image === null || a.image < a.source;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A command runner, for the two questions asked below; injected so the rules can be tested. */
|
||||||
|
export type Ask = (command: string, args: string[]) => { status: number | null; stdout: string };
|
||||||
|
|
||||||
|
const ask: Ask = (command, args) => {
|
||||||
|
const ran = spawnSync(command, args, { encoding: "utf8" });
|
||||||
|
return { status: ran.status, stdout: ran.stdout ?? "" };
|
||||||
|
};
|
||||||
|
|
||||||
|
/** ageOfImage is when the local image store made this tag, or null when it holds no such image. */
|
||||||
|
export function ageOfImage(tag: string, run: Ask = ask): number | null {
|
||||||
|
const ran = run("docker", ["image", "inspect", "--format", "{{.Created}}", tag]);
|
||||||
|
if (ran.status !== 0) return null;
|
||||||
|
const at = Date.parse(ran.stdout.trim());
|
||||||
|
return Number.isNaN(at) ? null : Math.floor(at / 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ageOfSource is the newest commit touching what the runtime is built from: the module's directory
|
||||||
|
* in the catalogue, and the whole of each repository it is built on top of. Uncommitted changes in
|
||||||
|
* any of them are newer than every commit.
|
||||||
|
*/
|
||||||
|
export function ageOfSource(catalogue: string, module: string, builtOn: string[], run: Ask = ask): number {
|
||||||
|
let newest = 0;
|
||||||
|
const at = (dir: string, path?: string): number => {
|
||||||
|
const args = ["-C", dir, "log", "-1", "--format=%ct"];
|
||||||
|
if (path) args.push("--", path);
|
||||||
|
const ran = run("git", args);
|
||||||
|
const t = Number.parseInt(ran.stdout.trim(), 10);
|
||||||
|
return ran.status === 0 && Number.isFinite(t) ? t : 0;
|
||||||
|
};
|
||||||
|
const dirty = (dir: string, path?: string): boolean => {
|
||||||
|
const args = ["-C", dir, "status", "--porcelain"];
|
||||||
|
if (path) args.push("--", path);
|
||||||
|
const ran = run("git", args);
|
||||||
|
return ran.status === 0 && ran.stdout.trim() !== "";
|
||||||
|
};
|
||||||
|
if (dirty(catalogue, `modules/${module}`)) return Infinity;
|
||||||
|
newest = Math.max(newest, at(catalogue, `modules/${module}`));
|
||||||
|
for (const dir of builtOn) {
|
||||||
|
if (dirty(dir)) return Infinity;
|
||||||
|
newest = Math.max(newest, at(dir));
|
||||||
|
}
|
||||||
|
return newest;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* plannedRuntimes is the runtime builds these beds need before they run, given where the run was
|
||||||
|
* pointed: nothing where no catalogue was named (the beds skip), one build per stale image
|
||||||
|
* otherwise. The repositories the runtime is built on are the siblings the build script itself
|
||||||
|
* reads (`MESH_TOOLS`, `MESH_SDK`, or the checkouts beside this one).
|
||||||
|
*/
|
||||||
|
export function plannedRuntimes(testFiles: string[], env: NodeJS.ProcessEnv = process.env,
|
||||||
|
root: string = process.cwd(), run: Ask = ask): Build[] {
|
||||||
|
const named = env["MESH_LAB_CATALOG"];
|
||||||
|
if (!named) return [];
|
||||||
|
const catalogue = catalogueRoot(named);
|
||||||
|
const builtOn = [
|
||||||
|
env["MESH_TOOLS"] ?? resolve(root, "..", "mesh-tools"),
|
||||||
|
env["MESH_SDK"] ?? resolve(root, "..", "mesh-sdk"),
|
||||||
|
];
|
||||||
|
const builds: Build[] = [];
|
||||||
|
for (const runtime of runtimesStockedBy(testFiles, root)) {
|
||||||
|
const ages: Ages = {
|
||||||
|
image: ageOfImage(runtime.tag, run),
|
||||||
|
source: ageOfSource(catalogue, runtime.module, builtOn, run),
|
||||||
|
};
|
||||||
|
if (!isStale(ages)) continue;
|
||||||
|
builds.push({
|
||||||
|
what: `runtime ${runtime.tag}`,
|
||||||
|
in: root,
|
||||||
|
argv: ["scripts/build-module-runtime.sh", runtime.module, join(tmpdir(), `mesh-lab-${runtime.module}.tar`)],
|
||||||
|
env: { MESH_CATALOG: catalogue, RUNTIME_TAG: runtime.tag },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return builds;
|
||||||
|
}
|
||||||
+1
-1
@@ -40,7 +40,7 @@ export async function runSuite(args: string[]): Promise<number> {
|
|||||||
if (!args.includes("--no-build")) {
|
if (!args.includes("--no-build")) {
|
||||||
// Before the run, always. The artifacts are built from two other repositories, and a suite
|
// Before the run, always. The artifacts are built from two other repositories, and a suite
|
||||||
// that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
|
// that tests yesterday's binary reports on code nobody is looking at (novox/hq 04-ISSUES/005).
|
||||||
const built = rebuild();
|
const built = rebuild(process.env, files);
|
||||||
if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
|
if (built.length > 0) console.log(`built: ${built.join(", ")}\n`);
|
||||||
}
|
}
|
||||||
// Read now, while it is true. The receipt names these, and reading them when the run ends
|
// Read now, while it is true. The receipt names these, and reading them when the run ends
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import { test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
|
||||||
|
import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A bed installs a catalogue module by reading the catalogue, never by carrying a copy.
|
||||||
|
*
|
||||||
|
* The beds used to build the manifests they install inline, as literals taken from the catalogue
|
||||||
|
* when each bed was written. The copies did not move when the catalogue did: six modules were
|
||||||
|
* converted to file-delivered secrets and not one bed ran the converted shape, because every bed
|
||||||
|
* ran its own copy (novox/hq 04-ISSUES/073). "Proven in the lab" then meant "the copy was proven".
|
||||||
|
*
|
||||||
|
* So: a manifest literal in a bed that names a catalogue module is refused, unless the bed is
|
||||||
|
* listed below with the reason it still carries one. The list is the debt, and it only shrinks.
|
||||||
|
*
|
||||||
|
* What this reads: `module: "<name>"` and `"module": "<name>"` with a `version` close by, either
|
||||||
|
* order, in test/integration/*.test.ts, against the catalogue's directory names. Skipped aloud
|
||||||
|
* where MESH_LAB_CATALOG is unset — a skip is reported, never silent. A bed that hid the name
|
||||||
|
* behind a computed string would pass — this is a fence, not a proof, and the reviewer of a bed
|
||||||
|
* that builds a manifest inline is the proof.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Beds that still carry an inline copy of a catalogue module's manifest, and why. Three reasons
|
||||||
|
* recur, and each names the work that removes the entry:
|
||||||
|
*
|
||||||
|
* BESIDE the catalogue's module CLAIMS the foundation's container (postgres claims mesh-store,
|
||||||
|
* lavinmq mesh-broker) and adopts it in place; the bed raises a second one beside the
|
||||||
|
* foundation's instead. Reading the catalogue changes what the bed raises — it would
|
||||||
|
* adopt — and the bed's assertions with it.
|
||||||
|
* WEARING the bed proves a mesh mechanism (a grant, a credential, a restart, a route) with a
|
||||||
|
* module cut down to the shape the mechanism needs — no upstream server, a secret in the
|
||||||
|
* environment, a requirement edge removed — and gives it a catalogue name. It is a mesh
|
||||||
|
* test wearing a catalogue module's name. It cannot simply be renamed: a module's name
|
||||||
|
* is its tool namespace and its broker scope, so a fixture running the module's runtime
|
||||||
|
* must carry the module's name (novox/hq ADR 0093). It reads the catalogue and installs
|
||||||
|
* what the module requires — the vault for a secret, the route module for a route — or
|
||||||
|
* it runs no real runtime and carries a name of its own.
|
||||||
|
* DIFFERS a module bed whose copy differs from the catalogue in more than the lab may rewrite
|
||||||
|
* (an image, a port, an address). Reading the catalogue is the fix and needs a run.
|
||||||
|
*/
|
||||||
|
const STILL_CARRIED: Record<string, { modules: string[]; why: string }> = {
|
||||||
|
"assigned-catalogue-apps.test.ts": { modules: ["postgres", "mongodb", "unifi", "marrytts"],
|
||||||
|
why: "BESIDE (postgres); DIFFERS (unifi takes its credentials from the environment, mongodb and marrytts drop listens)" },
|
||||||
|
"assigned-catalogue-media.test.ts": { modules: ["sonarr", "radarr"],
|
||||||
|
why: "DIFFERS: both drop the route requirement the catalogue declares, and take their API keys from the environment" },
|
||||||
|
"assigned-catalogue-small.test.ts": { modules: ["postgres", "minio", "redis", "plex"],
|
||||||
|
why: "BESIDE (postgres); DIFFERS (minio's root password by env-file, redis minting its own secret instead of the vault's, plex without its server)" },
|
||||||
|
"assigned-model-usage.test.ts": { modules: ["postgres"], why: "BESIDE" },
|
||||||
|
"assigned-two-node-db.test.ts": { modules: ["baserow", "letta"],
|
||||||
|
why: "DIFFERS: baserow drops its route requirement, letta drops its ports" },
|
||||||
|
"lavinmq-bed.test.ts": { modules: ["lavinmq", "amqp-ping"],
|
||||||
|
why: "BESIDE (lavinmq, with a bootstrap step and a data directory the catalogue has not got); DIFFERS (amqp-ping names its entrypoint)" },
|
||||||
|
};
|
||||||
|
|
||||||
|
const beds = resolve(import.meta.dirname, "integration");
|
||||||
|
|
||||||
|
test("a bed that installs a catalogue module reads the catalogue", (t) => {
|
||||||
|
const absent = catalogueIsPresent();
|
||||||
|
if (absent) {
|
||||||
|
// Said, not silent: a check that cannot see the catalogue has checked nothing.
|
||||||
|
t.skip(`cannot check — ${absent}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const names = new Set(readdirSync(catalogueDir(), { withFileTypes: true })
|
||||||
|
.filter((d) => d.isDirectory() && existsSync(resolve(catalogueDir(), d.name, "module.json")))
|
||||||
|
.map((d) => d.name));
|
||||||
|
|
||||||
|
const offences: string[] = [];
|
||||||
|
for (const file of readdirSync(beds).filter((f) => f.endsWith(".test.ts")).sort()) {
|
||||||
|
const text = readFileSync(resolve(beds, file), "utf8");
|
||||||
|
const found = new Set<string>();
|
||||||
|
// A manifest literal: the module's name with its version close behind it. A `module:` key
|
||||||
|
// elsewhere (a table of what to register, a grant entry) has no version and is not one.
|
||||||
|
for (const m of text.matchAll(/(?:^|[\s{,])(?:"module"|module)\s*:\s*"([a-z0-9.-]+)"[^}]{0,160}?(?:"version"|version)\s*:/g)) {
|
||||||
|
if (names.has(m[1]!)) found.add(m[1]!);
|
||||||
|
}
|
||||||
|
// And the other order — a literal that names its version first.
|
||||||
|
for (const m of text.matchAll(/(?:^|[\s{,])(?:"version"|version)\s*:\s*"[^"]*"[^}]{0,160}?(?:"module"|module)\s*:\s*"([a-z0-9.-]+)"/g)) {
|
||||||
|
if (names.has(m[1]!)) found.add(m[1]!);
|
||||||
|
}
|
||||||
|
const declared = STILL_CARRIED[file];
|
||||||
|
for (const name of [...found].sort()) {
|
||||||
|
if (declared?.modules.includes(name)) continue;
|
||||||
|
offences.push(`${file}: an inline manifest for the catalogue's '${name}'`);
|
||||||
|
}
|
||||||
|
for (const name of declared?.modules ?? []) {
|
||||||
|
if (!found.has(name)) offences.push(`${file}: declared as still carrying '${name}', and it does not — remove the declaration`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.deepEqual(offences, [],
|
||||||
|
`a bed carries a copy of a catalogue manifest; read it with catalogueModule() from the harness:\n ${offences.join("\n ")}`);
|
||||||
|
});
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
/**
|
||||||
|
* **A module that takes a shared-cache grant presents the login it was granted** (novox/hq 081).
|
||||||
|
*
|
||||||
|
* The cache provider scopes each consumer to an ACL user of its own, confined to keys under its
|
||||||
|
* login (novox/hq 080). A consumer that hands its software the password and not the login logs in
|
||||||
|
* as the server's default user: the grant is honoured by the provider and ignored by the consumer,
|
||||||
|
* and nothing notices while the default user is open. The grant bed proves the provider's half
|
||||||
|
* against a consumer written to the contract; this holds every catalogue module that asks for the
|
||||||
|
* cache to its half — the login, as `${bound:redis-cache:as}`, somewhere it hands its software.
|
||||||
|
*
|
||||||
|
* What it cannot see is whether the software also keeps its keys under that login: that is the
|
||||||
|
* software's, and a module whose software cannot (fixed key or channel names in its code) does not
|
||||||
|
* take the shared cache at all — baserow runs its own, and n8n in its shipped mode needs none.
|
||||||
|
*/
|
||||||
|
import { test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
|
||||||
|
import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
|
||||||
|
|
||||||
|
const CACHE = "redis-cache";
|
||||||
|
|
||||||
|
/** What a module hands its software: every file it writes, and every container's environment and
|
||||||
|
* arguments. A comment, a `why`, or a declared exception is not handed to anything. */
|
||||||
|
function handedToSoftware(manifest: string): string[] {
|
||||||
|
const m = JSON.parse(manifest) as { resources?: Record<string, unknown>[] };
|
||||||
|
const out: string[] = [];
|
||||||
|
for (const r of m.resources ?? []) {
|
||||||
|
if (typeof r["content"] === "string") out.push(r["content"] as string);
|
||||||
|
if (r["env"] && typeof r["env"] === "object") out.push(...Object.values(r["env"] as Record<string, string>).map(String));
|
||||||
|
if (Array.isArray(r["args"])) out.push(...(r["args"] as unknown[]).map(String));
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a manifest hands its software the login it is granted for the cache. */
|
||||||
|
function presentsTheLogin(manifest: string): boolean {
|
||||||
|
const login = `\${bound:${CACHE}:as}`;
|
||||||
|
return handedToSoftware(manifest).some((given) => given.includes(login));
|
||||||
|
}
|
||||||
|
|
||||||
|
test("the check sees a module that hands its software the password and not the login", () => {
|
||||||
|
const passwordOnly = JSON.stringify({ module: "m", requires: [CACHE], resources: [
|
||||||
|
{ id: "env", type: "file", path: "/x", content: `HOST=\${bound:${CACHE}:at}\nPASSWORD=\${secret:${CACHE}}\n` }] });
|
||||||
|
const withLogin = JSON.stringify({ module: "m", requires: [CACHE], resources: [
|
||||||
|
{ id: "env", type: "file", path: "/x", content: `USER=\${bound:${CACHE}:as}\nPASSWORD=\${secret:${CACHE}}\n` }] });
|
||||||
|
assert.equal(presentsTheLogin(passwordOnly), false);
|
||||||
|
assert.equal(presentsTheLogin(withLogin), true);
|
||||||
|
// Named only where no software reads it — a declared reason — is not presenting it.
|
||||||
|
const onlyInAReason = JSON.stringify({ module: "m", requires: [CACHE], resources: [
|
||||||
|
{ id: "srv", type: "container", name: "s", image: "x", env: { PASSWORD: `\${secret:${CACHE}}` },
|
||||||
|
"secrets-in-environment": `the login is \${bound:${CACHE}:as}` }] });
|
||||||
|
assert.equal(presentsTheLogin(onlyInAReason), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("every catalogue module that takes the shared cache presents the login it was granted", (t) => {
|
||||||
|
const absent = catalogueIsPresent();
|
||||||
|
if (absent) {
|
||||||
|
t.skip(`cannot check — ${absent}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const offences: string[] = [];
|
||||||
|
for (const d of readdirSync(catalogueDir(), { withFileTypes: true })) {
|
||||||
|
const file = resolve(catalogueDir(), d.name, "module.json");
|
||||||
|
if (!d.isDirectory() || !existsSync(file)) continue;
|
||||||
|
const text = readFileSync(file, "utf8");
|
||||||
|
const m = JSON.parse(text) as { requires?: string[] };
|
||||||
|
if (!(m.requires ?? []).includes(CACHE)) continue;
|
||||||
|
if (!presentsTheLogin(text)) offences.push(d.name);
|
||||||
|
}
|
||||||
|
assert.deepEqual(offences, [],
|
||||||
|
`these take the shared cache and never hand their software the login (\${bound:${CACHE}:as}), so they ` +
|
||||||
|
`log in as the server's default user — present the login, or run a cache of their own:\n ${offences.join("\n ")}`);
|
||||||
|
});
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import { test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { catalogueModule } from "./integration/harness.ts";
|
||||||
|
import type { HeldImage } from "../src/pinning.ts";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The shared loader thirteen beds install through (novox/hq 04-ISSUES/073, ADR 0089): it reads
|
||||||
|
* the catalogue's manifest and rewrites only what the lab must. Checked here against a catalogue
|
||||||
|
* written by the test, so the rules hold without a lab run.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const digest = (c: string) => "sha256:" + c.repeat(64);
|
||||||
|
const held: HeldImage[] = [
|
||||||
|
{ requested: "mesh-runtime-thing:development", repository: "mesh-runtime-thing", reference: digest("a") },
|
||||||
|
{ requested: "mesh-helper:development", repository: "mesh-helper", reference: digest("b") },
|
||||||
|
];
|
||||||
|
|
||||||
|
function aCatalogueWith(manifest: object): () => void {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "mesh-lab-catalogue-"));
|
||||||
|
mkdirSync(join(root, "modules", "distribution"), { recursive: true });
|
||||||
|
writeFileSync(join(root, "modules", "distribution", "module.json"), "{}");
|
||||||
|
mkdirSync(join(root, "modules", "thing"));
|
||||||
|
writeFileSync(join(root, "modules", "thing", "module.json"), JSON.stringify(manifest));
|
||||||
|
const before = process.env["MESH_LAB_CATALOG"];
|
||||||
|
process.env["MESH_LAB_CATALOG"] = root;
|
||||||
|
return () => {
|
||||||
|
if (before === undefined) delete process.env["MESH_LAB_CATALOG"]; else process.env["MESH_LAB_CATALOG"] = before;
|
||||||
|
rmSync(root, { recursive: true, force: true });
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const thing = {
|
||||||
|
module: "thing", version: "1",
|
||||||
|
resources: [
|
||||||
|
{ id: "server", type: "container", name: "thing", image: "postgres@" + digest("c"), ports: ["8080", "9090:9090"], env: { A: "1" } },
|
||||||
|
{ id: "runtime", type: "container", name: "mesh-thing", artifact: "runtime" },
|
||||||
|
],
|
||||||
|
build: { artifacts: [{ name: "runtime", kind: "image", from: "Dockerfile" }] },
|
||||||
|
};
|
||||||
|
|
||||||
|
test("the runtime artifact becomes the image the machine holds, and the build section goes", () => {
|
||||||
|
const restore = aCatalogueWith(thing);
|
||||||
|
try {
|
||||||
|
const m = JSON.parse(catalogueModule("thing", held)) as { build?: unknown; resources: Record<string, unknown>[] };
|
||||||
|
assert.equal(m.build, undefined);
|
||||||
|
const runtime = m.resources.find((r) => r["id"] === "runtime")!;
|
||||||
|
assert.equal(runtime["image"], digest("a"));
|
||||||
|
assert.equal(runtime["artifact"], undefined);
|
||||||
|
// An image already pinned to a digest passes through as written.
|
||||||
|
assert.equal(m.resources.find((r) => r["id"] === "server")!["image"], "postgres@" + digest("c"));
|
||||||
|
} finally { restore(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an artifact the bed did not name is refused, and a named one resolves to the stocked image", () => {
|
||||||
|
const helper = { ...thing, resources: [{ id: "helper", type: "container", name: "h", artifact: "helper" }] };
|
||||||
|
const restore = aCatalogueWith(helper);
|
||||||
|
try {
|
||||||
|
assert.throws(() => catalogueModule("thing", held), /names the "helper" artifact/);
|
||||||
|
const m = JSON.parse(catalogueModule("thing", held, { artifacts: { helper: "mesh-helper" } })) as { resources: Record<string, unknown>[] };
|
||||||
|
assert.equal(m.resources[0]!["image"], digest("b"));
|
||||||
|
assert.throws(() => catalogueModule("thing", held, { artifacts: { helper: "mesh-nothing" } }), /stocked no such image/);
|
||||||
|
} finally { restore(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a host-port remap and a lab address are the only other things that change", () => {
|
||||||
|
const restore = aCatalogueWith(thing);
|
||||||
|
try {
|
||||||
|
const m = JSON.parse(catalogueModule("thing", held, {
|
||||||
|
ports: { "8080": "8090:8080" },
|
||||||
|
env: { server: { B: "2" } },
|
||||||
|
})) as { resources: Record<string, unknown>[] };
|
||||||
|
const server = m.resources.find((r) => r["id"] === "server")!;
|
||||||
|
assert.deepEqual(server["ports"], ["8090:8080", "9090:9090"]);
|
||||||
|
assert.deepEqual(server["env"], { A: "1", B: "2" });
|
||||||
|
} finally { restore(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a manifest the catalogue does not have is refused by name", () => {
|
||||||
|
const restore = aCatalogueWith(thing);
|
||||||
|
try {
|
||||||
|
assert.throws(() => catalogueModule("nothing", held), /no manifest for nothing/);
|
||||||
|
} finally { restore(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an upstream artifact resolves to the reference the manifest pins, as the machine pulls it", () => {
|
||||||
|
const web = {
|
||||||
|
module: "thing", version: "1",
|
||||||
|
resources: [{ id: "server", type: "container", name: "web", artifact: "server" }],
|
||||||
|
build: { artifacts: [{ name: "server", kind: "upstream", from: "alpine@" + digest("e") }] },
|
||||||
|
};
|
||||||
|
const restore = aCatalogueWith(web);
|
||||||
|
try {
|
||||||
|
const m = JSON.parse(catalogueModule("thing", held)) as { resources: Record<string, unknown>[] };
|
||||||
|
assert.equal(m.resources[0]!["image"], "alpine@" + digest("e"));
|
||||||
|
assert.equal(m.resources[0]!["artifact"], undefined);
|
||||||
|
} finally { restore(); }
|
||||||
|
});
|
||||||
@@ -21,29 +21,25 @@
|
|||||||
*/
|
*/
|
||||||
import { test, before, after } from "node:test";
|
import { test, before, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { dirname, resolve } from "node:path";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
const binary = hostBinaryPath();
|
const binary = hostBinaryPath();
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
|
||||||
const skip = !capability.usable ? `lab not usable: ${capability.why}`
|
const skip = !capability.usable ? `lab not usable: ${capability.why}`
|
||||||
: !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
: !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
|
: !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "adopted-store-cross-node";
|
const SCENARIO = "adopted-store-cross-node";
|
||||||
const NODE = "node2";
|
const NODE = "node2";
|
||||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
|
||||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
|
||||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
|
||||||
|
|
||||||
let instanceId = "";
|
let instanceId = "";
|
||||||
let held: HeldImage[] = [];
|
let held: HeldImage[] = [];
|
||||||
@@ -65,29 +61,12 @@ async function must(machine: string, command: string, timeoutMs?: number): Promi
|
|||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||||
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||||
}
|
}
|
||||||
function pinned(reference: string): string { return onTheMachine(reference, held); }
|
|
||||||
function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); }
|
function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); }
|
||||||
|
|
||||||
/** Load a committed module.json with its container images rewritten to the scenario's pinned digests. */
|
/** The catalogue's manifest as the lab runs it (harness), and whether it needs a broker account. */
|
||||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||||
const path = resolve(catalogDir, name, "module.json");
|
const manifest = catalogueModule(name, held);
|
||||||
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
return { manifest, broker: needsBrokerAccount(manifest) };
|
||||||
resources?: { type: string; image?: string; artifact?: string }[];
|
|
||||||
};
|
|
||||||
for (const r of m.resources ?? []) {
|
|
||||||
if (r.type !== "container") continue;
|
|
||||||
if (typeof r.image === "string") {
|
|
||||||
// A placeholder image (mesh-runtime-<m>@0…0) resolves to the stocked digest, as redis does.
|
|
||||||
r.image = pinned(r.image);
|
|
||||||
} else if (typeof r.artifact === "string") {
|
|
||||||
// The bundle-model bed does not build, so resolve a module's runtime ARTIFACT to its stocked
|
|
||||||
// image directly — postgres/lavinmq name their provisioner by artifact, not a placeholder.
|
|
||||||
r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`);
|
|
||||||
delete r.artifact;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const manifest = JSON.stringify(m);
|
|
||||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
|
||||||
}
|
}
|
||||||
function tokenFrom(said: string): string {
|
function tokenFrom(said: string): string {
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||||
@@ -105,7 +84,6 @@ async function install(name: string, node: string): Promise<void> {
|
|||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
if (skip) return;
|
if (skip) return;
|
||||||
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`) });
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`) });
|
||||||
instanceId = raised.instanceId;
|
instanceId = raised.instanceId;
|
||||||
held = raised.images;
|
held = raised.images;
|
||||||
|
|||||||
@@ -0,0 +1,976 @@
|
|||||||
|
/**
|
||||||
|
* A MACHINE IN USE IS ADOPTED BEFORE IT IS CONVERGED (novox/hq ADR 0100, and ADR 0101 on what
|
||||||
|
* "in use" ignores).
|
||||||
|
*
|
||||||
|
* The mesh replaces a predecessor running on the same machines. This bed prepares a machine the
|
||||||
|
* way the predecessor leaves one — the record's own words, "How it is checked":
|
||||||
|
*
|
||||||
|
* - its firewall (ufw) allowing a served port and denying the rest, incoming and routed, with the
|
||||||
|
* predecessor's published container ports filtered through it (the ufw-docker arrangement);
|
||||||
|
* - a service container, `hello-web`, listening on that port under a name the catalogue's
|
||||||
|
* `hello-web` module also uses, and a file at a path that module declares;
|
||||||
|
* - a stand-in for the predecessor's control that rewrites that file, stopped by the operator
|
||||||
|
* before adoption as the record prescribes, and started again later to play one forgotten;
|
||||||
|
* - a container holding the registry's port.
|
||||||
|
*
|
||||||
|
* Then it asks, in the record's order: a converged genesis refuses; an adopted one refuses the held
|
||||||
|
* registry port and comes up on another; nothing that serves changed; the store is unreachable
|
||||||
|
* from outside and reachable where it must be; the mesh works through the found firewall, across a
|
||||||
|
* reload and a reboot; a predecessor still writing is caught; assigning prepares and taking cuts
|
||||||
|
* over; converging previews, refuses while a found container is held, flips, and returns.
|
||||||
|
*
|
||||||
|
* **The module under migration is the catalogue's `hello-web` with its route requirement taken
|
||||||
|
* off**, read from the catalogue (never a copy): the route needs a proxy module and a public name,
|
||||||
|
* neither of which is what adoption is about. Its names — the container, the file, the port — are
|
||||||
|
* the catalogue's, which is the point: they are what the predecessor also uses.
|
||||||
|
*
|
||||||
|
* **The forge is in the lab.** Genesis builds the control plane and the catalogue from a
|
||||||
|
* repository and a commit; this bed serves the checkouts it was pointed at (their HEADs) from the
|
||||||
|
* `outsider` machine, so the run builds exactly the code under test and nothing on the workstation
|
||||||
|
* listens for the lab.
|
||||||
|
*
|
||||||
|
* Each step is recorded rather than allowed to throw; a step whose dependency failed is not
|
||||||
|
* attempted, and the report says which.
|
||||||
|
*
|
||||||
|
* MESH_LAB_INCUS='sudo -n incus'
|
||||||
|
* MESH_LAB_HOST_BINARY=<mesh-host>/mesh-host MESH_LAB_BOOTSTRAP_BINARY=<mesh-host>/mesh-bootstrap
|
||||||
|
* MESH_LAB_BUNDLE=<mesh-host>/examples/foundation-first-node.lock
|
||||||
|
* MESH_LAB_CATALOG=<mesh-catalog>/modules MESH_LAB_MODULES=<mesh-controller>/examples/modules
|
||||||
|
* MESH_TOOLS=<mesh-tools> MESH_SDK=<mesh-sdk> (default: the checkouts beside this one)
|
||||||
|
* MESH_LAB_KEEP=1 leave it standing MESH_LAB_WARM=1 iterate from the adopted foundation
|
||||||
|
*/
|
||||||
|
import { test, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||||
|
import { execFileSync } from "node:child_process";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { dirname, join, resolve } from "node:path";
|
||||||
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
|
import { destroy, exec, push, instanceNameOf } from "../../src/lifecycle/operate.ts";
|
||||||
|
import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
|
import { waitUntilAllUsable } from "../../src/lifecycle/ready.ts";
|
||||||
|
import { incus } from "../../src/incus/client.ts";
|
||||||
|
import { catalogueRoot } from "../../src/repos.ts";
|
||||||
|
import { ready, returnTo, keep } from "../../src/warm.ts";
|
||||||
|
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueManifest } from "./harness.ts";
|
||||||
|
import { genesis, type GenesisOptions } from "./genesis.ts";
|
||||||
|
|
||||||
|
const SCENARIO = "adoption";
|
||||||
|
const CONTROL = "anchor";
|
||||||
|
const JOINER = "joiner";
|
||||||
|
const OUTSIDER = "outsider";
|
||||||
|
const ANCHOR = "192.0.2.10";
|
||||||
|
const JOINER_ADDRESS = "192.0.2.20";
|
||||||
|
const FORGE = "192.0.2.30";
|
||||||
|
|
||||||
|
/** The port the predecessor serves, and the module that also names its container and file. */
|
||||||
|
const SERVED = 8080;
|
||||||
|
const SERVICE = "hello-web";
|
||||||
|
const SERVICE_FILE = "/var/lib/hello-web/index.html";
|
||||||
|
const PREDECESSOR_PAGE = "hello from the predecessor\n";
|
||||||
|
/** The registry's port, which the predecessor holds — and the one the mesh is given instead. */
|
||||||
|
const HELD_REGISTRY = 5000;
|
||||||
|
const REGISTRY_PORT = 5100;
|
||||||
|
const STORE_PORT = 5432;
|
||||||
|
const BUS_PORT = 5671;
|
||||||
|
const HUB_PORT = 51820;
|
||||||
|
const NETWORK_MODULE = "networking";
|
||||||
|
const FILTER_MODULE = "nftables";
|
||||||
|
|
||||||
|
/** Upstream images, pinned as the catalogue pins them (the harness's table). */
|
||||||
|
const ALPINE = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b";
|
||||||
|
const REGISTRY_IMAGE = "registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
|
||||||
|
|
||||||
|
const capability = await labIsUsable();
|
||||||
|
const binary = hostBinaryPath();
|
||||||
|
const installer = bootstrapBinaryPath();
|
||||||
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
|
const catalogDir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||||
|
const modulesDir = process.env["MESH_LAB_MODULES"] ?? "";
|
||||||
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||||
|
const WARM = !!process.env["MESH_LAB_WARM"];
|
||||||
|
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "adoption-live" : undefined);
|
||||||
|
|
||||||
|
/** The checkouts this run builds from, served by the lab's forge. */
|
||||||
|
const REPOS: Record<string, string> = {
|
||||||
|
"mesh-controller": modulesDir ? dirname(dirname(modulesDir)) : "",
|
||||||
|
"mesh-catalog": catalogDir ? catalogueRoot(catalogDir) : "",
|
||||||
|
"mesh-tools": process.env["MESH_TOOLS"] ?? resolve(process.cwd(), "..", "mesh-tools"),
|
||||||
|
"mesh-sdk": process.env["MESH_SDK"] ?? resolve(process.cwd(), "..", "mesh-sdk"),
|
||||||
|
};
|
||||||
|
|
||||||
|
const skip =
|
||||||
|
!capability.usable ? capability.why :
|
||||||
|
!binary ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" :
|
||||||
|
!installer ? "MESH_LAB_BOOTSTRAP_BINARY is not set to a built mesh-bootstrap" :
|
||||||
|
!bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation template" :
|
||||||
|
!catalogDir || !existsSync(catalogDir) ? "MESH_LAB_CATALOG is not set to mesh-catalog/modules" :
|
||||||
|
!modulesDir ? "MESH_LAB_MODULES is not set, so there is no control-plane checkout to build from" :
|
||||||
|
Object.entries(REPOS).find(([, p]) => !p || !existsSync(resolve(p, ".git")))
|
||||||
|
?.map((x) => `no checkout of ${x[0]} at ${x[1]}`)[0] ?? false;
|
||||||
|
|
||||||
|
let instanceId = "";
|
||||||
|
|
||||||
|
// ---- talking to the machines ------------------------------------------------------------------
|
||||||
|
|
||||||
|
function quote(s: string): string {
|
||||||
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||||
|
}
|
||||||
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||||
|
const { stdout } = await exec(instanceId, machine, [
|
||||||
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||||
|
], timeoutMs);
|
||||||
|
const marker = stdout.lastIndexOf("__exit=");
|
||||||
|
if (marker < 0) return { out: stdout, ok: false };
|
||||||
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||||
|
}
|
||||||
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
const { out, ok } = await on(machine, command, timeoutMs);
|
||||||
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
/** The control plane, retried across the brief windows in which the mesh recreates it. */
|
||||||
|
async function meshSays(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||||
|
const deadline = Date.now() + (timeoutMs ?? 120_000);
|
||||||
|
for (;;) {
|
||||||
|
const r = await on(CONTROL, `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||||
|
if (r.ok) return r;
|
||||||
|
if (/is not running|No such container|No such exec instance|Cannot connect to the Docker daemon|is restarting/i.test(r.out) &&
|
||||||
|
Date.now() < deadline) {
|
||||||
|
await sleep(2_000);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
const r = await meshSays(command, timeoutMs);
|
||||||
|
if (!r.ok) throw new Error(`${CONTROL}: mesh-controller ${command}\n${r.out}`);
|
||||||
|
return r.out;
|
||||||
|
}
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((r) => setTimeout(r, ms));
|
||||||
|
}
|
||||||
|
/** Poll until `probe` returns a value, or fail naming the last thing it saw. */
|
||||||
|
async function until<T>(what: string, seconds: number, probe: () => Promise<T | null>, last: () => string): Promise<T> {
|
||||||
|
const deadline = Date.now() + seconds * 1000;
|
||||||
|
for (;;) {
|
||||||
|
const got = await probe();
|
||||||
|
if (got !== null) return got;
|
||||||
|
if (Date.now() > deadline) throw new Error(`${what} — not within ${seconds}s. Last:\n${last()}`);
|
||||||
|
await sleep(5_000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/** Whether a TCP connection from `machine` to address:port opens within three seconds. */
|
||||||
|
async function connects(machine: string, address: string, port: number): Promise<boolean> {
|
||||||
|
return (await on(machine, `timeout 4 bash -c ${quote(`</dev/tcp/${address}/${port}`)}`)).ok;
|
||||||
|
}
|
||||||
|
/** Register a module with the control plane from a manifest's text. */
|
||||||
|
async function registerModule(module: string, manifest: string): Promise<string> {
|
||||||
|
const local = join(tmpdir(), `mesh-lab-adoption-${process.pid}-${module}.json`);
|
||||||
|
writeFileSync(local, manifest);
|
||||||
|
await push(instanceId, CONTROL, local, `/tmp/${module}.json`);
|
||||||
|
await must(CONTROL, `docker cp /tmp/${module}.json mesh-controller:/${module}.json`);
|
||||||
|
return mesh(`module add /${module}.json`);
|
||||||
|
}
|
||||||
|
async function nodeShow(node: string): Promise<string> {
|
||||||
|
return mesh(`node show ${node}`);
|
||||||
|
}
|
||||||
|
/** The anchor's address on the private network. */
|
||||||
|
async function meshAddressOf(machine: string): Promise<string> {
|
||||||
|
const out = await must(machine, `ip -4 -o addr show dev mesh0`);
|
||||||
|
const found = out.match(/inet (\d+\.\d+\.\d+\.\d+)\//)?.[1];
|
||||||
|
assert.ok(found, `${machine} has no address on mesh0:\n${out}`);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
/** The rules ufw was given, one per line, as `ufw show added` prints them. */
|
||||||
|
async function ufwAdded(): Promise<string[]> {
|
||||||
|
const out = await must(CONTROL, `ufw show added`);
|
||||||
|
return out.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
|
||||||
|
}
|
||||||
|
function marked(rule: string): boolean {
|
||||||
|
return /comment 'mesh-host /.test(rule);
|
||||||
|
}
|
||||||
|
async function restartMachine(machine: string): Promise<void> {
|
||||||
|
const name = await instanceNameOf(instanceId, machine);
|
||||||
|
await incus(["restart", name], 180_000);
|
||||||
|
await waitUntilAllUsable([name], 300, (m) => console.log(` restart: ${m}`));
|
||||||
|
}
|
||||||
|
/** Until the anchor reports what it was last sent as applied and current. */
|
||||||
|
async function settled(node = CONTROL, withinMs = 300_000): Promise<void> {
|
||||||
|
let last = "";
|
||||||
|
await until(`${node} reports the declaration it was sent as applied and current`, withinMs / 1000, async () => {
|
||||||
|
const asked = await meshSays(`status --json`);
|
||||||
|
last = asked.out;
|
||||||
|
if (!asked.ok) return null;
|
||||||
|
try {
|
||||||
|
const state = JSON.parse(asked.out) as {
|
||||||
|
wrong: { node: string; outcome: string }[];
|
||||||
|
waiting: { node: string }[];
|
||||||
|
reported: { node: string; outcome: string; current: boolean }[];
|
||||||
|
};
|
||||||
|
const bad = state.wrong.find((w) => w.node === node);
|
||||||
|
if (bad) throw new Error(`${node} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
||||||
|
const word = state.reported.find((r) => r.node === node);
|
||||||
|
return !state.waiting.some((w) => w.node === node) && word?.outcome === "applied" && word.current ? true : null;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}, () => last);
|
||||||
|
}
|
||||||
|
/** Send a node what it should be, and wait until it says it applied it. */
|
||||||
|
async function pushAndSettle(node: string): Promise<string> {
|
||||||
|
const said = await mesh(`push ${node}`, 600_000);
|
||||||
|
await settled(node);
|
||||||
|
return said;
|
||||||
|
}
|
||||||
|
function tokenFrom(said: string): string {
|
||||||
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||||
|
assert.ok(found, `no token in:\n${said}`);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the lab's forge ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serve the checkouts under test from the outsider machine, over git's own protocol.
|
||||||
|
*
|
||||||
|
* Each checkout's HEAD is pushed into a bare repository as `main` and `lab`, the lot is carried in,
|
||||||
|
* and a git daemon answers on the outsider's scenario address — which the anchor's builder reaches
|
||||||
|
* over the hosting segment. Returns the commit each repository is served at.
|
||||||
|
*/
|
||||||
|
async function raiseForge(): Promise<Record<string, string>> {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "mesh-lab-forge-"));
|
||||||
|
const heads: Record<string, string> = {};
|
||||||
|
try {
|
||||||
|
for (const [name, checkout] of Object.entries(REPOS)) {
|
||||||
|
const bare = join(dir, `${name}.git`);
|
||||||
|
execFileSync("git", ["init", "-q", "--bare", bare]);
|
||||||
|
execFileSync("git", ["-C", checkout, "push", "-q", "--force", bare,
|
||||||
|
"HEAD:refs/heads/main", "HEAD:refs/heads/lab"], { stdio: "pipe" });
|
||||||
|
heads[name] = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
|
||||||
|
}
|
||||||
|
const tar = join(tmpdir(), `mesh-lab-forge-${process.pid}.tar`);
|
||||||
|
execFileSync("tar", ["-cf", tar, "-C", dir, "."]);
|
||||||
|
await push(instanceId, OUTSIDER, tar, "/tmp/forge.tar");
|
||||||
|
rmSync(tar, { force: true });
|
||||||
|
} finally {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
await must(OUTSIDER, `command -v git >/dev/null || pacman -S --noconfirm --needed git`, 600_000);
|
||||||
|
// Owned by the daemon's user: extracted as the workstation's uid, git refuses to serve a
|
||||||
|
// repository someone else owns ("dubious ownership"), and the clone fails with no reason given.
|
||||||
|
await must(OUTSIDER, `mkdir -p /srv/git && tar --no-same-owner -xf /tmp/forge.tar -C /srv/git && chown -R root:root /srv/git && ` +
|
||||||
|
`git daemon --base-path=/srv/git --export-all --reuseaddr --detach --listen=${FORGE} --pid-file=/run/git-daemon.pid`);
|
||||||
|
await must(OUTSIDER, `git ls-remote git://${FORGE}/mesh-controller.git lab`);
|
||||||
|
await until("the lab's forge answers the anchor", 60, async () =>
|
||||||
|
(await connects(CONTROL, FORGE, 9418)) ? true : null, () => "no connection to 9418");
|
||||||
|
return heads;
|
||||||
|
}
|
||||||
|
const forgeUrl = (repo: string) => `git://${FORGE}/${repo}.git`;
|
||||||
|
|
||||||
|
// ---- the predecessor ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The ufw-docker arrangement: published container ports pass through ufw's route rules, and
|
||||||
|
* traffic from private ranges is let through. It is how a ufw machine filters what docker publishes
|
||||||
|
* at all — without it docker's own rules bypass ufw entirely (novox/hq research 012 measured 52
|
||||||
|
* forwarding rules on the control-node, one per served port).
|
||||||
|
*/
|
||||||
|
const UFW_DOCKER = `
|
||||||
|
# BEGIN UFW AND DOCKER
|
||||||
|
*filter
|
||||||
|
:ufw-user-forward - [0:0]
|
||||||
|
:ufw-docker-logging-deny - [0:0]
|
||||||
|
:DOCKER-USER - [0:0]
|
||||||
|
-A DOCKER-USER -j ufw-user-forward
|
||||||
|
-A DOCKER-USER -j RETURN -s 10.0.0.0/8
|
||||||
|
-A DOCKER-USER -j RETURN -s 172.16.0.0/12
|
||||||
|
-A DOCKER-USER -j RETURN -s 192.168.0.0/16
|
||||||
|
-A DOCKER-USER -p udp -m udp --sport 53 --dport 1024:65535 -j RETURN
|
||||||
|
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 192.168.0.0/16
|
||||||
|
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 10.0.0.0/8
|
||||||
|
-A DOCKER-USER -j ufw-docker-logging-deny -p tcp -m tcp --tcp-flags FIN,SYN,RST,ACK SYN -d 172.16.0.0/12
|
||||||
|
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 192.168.0.0/16
|
||||||
|
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 10.0.0.0/8
|
||||||
|
-A DOCKER-USER -j ufw-docker-logging-deny -p udp -m udp --dport 0:32767 -d 172.16.0.0/12
|
||||||
|
-A DOCKER-USER -j RETURN
|
||||||
|
-A ufw-docker-logging-deny -j DROP
|
||||||
|
COMMIT
|
||||||
|
# END UFW AND DOCKER
|
||||||
|
`;
|
||||||
|
|
||||||
|
/** The stand-in for the predecessor's configuration sync: it rewrites the file every ten seconds. */
|
||||||
|
const STAND_IN = `[Unit]
|
||||||
|
Description=Stand-in for the predecessor's configuration sync: rewrites a file a catalogue module declares
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
ExecStart=/bin/sh -c 'while true; do printf "hello from the predecessor, synced %%s\\\\n" "$(date +%%s)" > ${SERVICE_FILE}; sleep 10; done'
|
||||||
|
`;
|
||||||
|
|
||||||
|
/** The page the predecessor's service loop serves — the catalogue module's own loop, verbatim. */
|
||||||
|
const SERVE_LOOP =
|
||||||
|
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done";
|
||||||
|
|
||||||
|
/** Where the bed keeps what the machine looked like before the mesh arrived — on the machine, so a warm restore keeps it. */
|
||||||
|
const BEFORE = "/root/predecessor";
|
||||||
|
|
||||||
|
/** A predecessor container with no restart policy: a runtime restart would lose it. */
|
||||||
|
const NO_POLICY = "predecessor-nopolicy";
|
||||||
|
|
||||||
|
/** The broker's plaintext port: published on every interface, and the filter admits it from the mesh only. */
|
||||||
|
const AMQP_PORT = 5672;
|
||||||
|
|
||||||
|
async function preparePredecessor(): Promise<string> {
|
||||||
|
const said: string[] = [];
|
||||||
|
await must(CONTROL, `pacman -S --noconfirm --needed ufw`, 600_000);
|
||||||
|
const rules = join(tmpdir(), `mesh-lab-ufw-docker-${process.pid}`);
|
||||||
|
writeFileSync(rules, UFW_DOCKER);
|
||||||
|
await push(instanceId, CONTROL, rules, "/tmp/ufw-docker.rules");
|
||||||
|
await must(CONTROL, [
|
||||||
|
`grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules || cat /tmp/ufw-docker.rules >> /etc/ufw/after.rules`,
|
||||||
|
`ufw default deny incoming`,
|
||||||
|
`ufw default allow outgoing`,
|
||||||
|
`ufw default deny routed`,
|
||||||
|
`ufw allow 22/tcp`,
|
||||||
|
`ufw allow ${SERVED}/tcp`,
|
||||||
|
`ufw route allow proto tcp from any to any port ${SERVED}`,
|
||||||
|
`ufw --force enable`,
|
||||||
|
`systemctl enable ufw`,
|
||||||
|
].join(" && "));
|
||||||
|
said.push(` firewall ufw: deny incoming and routed; allow 22 and ${SERVED}; ufw-docker after.rules`);
|
||||||
|
|
||||||
|
await must(CONTROL, `mkdir -p /var/lib/hello-web && printf %s ${quote(PREDECESSOR_PAGE)} > ${SERVICE_FILE}`);
|
||||||
|
await must(CONTROL,
|
||||||
|
`docker run -d --name ${SERVICE} --restart unless-stopped -p ${SERVED}:${SERVED} ` +
|
||||||
|
`-v ${SERVICE_FILE}:/www/index.html:ro ${ALPINE} sh -c ${quote(SERVE_LOOP)}`, 600_000);
|
||||||
|
await must(CONTROL,
|
||||||
|
`docker run -d --name predecessor-registry --restart unless-stopped -p ${HELD_REGISTRY}:5000 ${REGISTRY_IMAGE}`, 600_000);
|
||||||
|
// A container the predecessor left running with no restart policy: a restart of the runtime
|
||||||
|
// would not bring it back, which is what ADR 0102 forbids the mesh from causing.
|
||||||
|
await must(CONTROL, `docker run -d --name ${NO_POLICY} ${ALPINE} sleep infinity`, 600_000);
|
||||||
|
// And a setting of the machine's own in the runtime's file, beside the registries it ships with.
|
||||||
|
await must(CONTROL,
|
||||||
|
`python3 - <<'EOF'
|
||||||
|
import json
|
||||||
|
f="/etc/docker/daemon.json"
|
||||||
|
d=json.load(open(f))
|
||||||
|
d["log-opts"]={"max-size":"7m"}
|
||||||
|
json.dump(d,open(f,"w"),indent=2)
|
||||||
|
EOF
|
||||||
|
systemctl reload docker`);
|
||||||
|
said.push(` containers ${SERVICE} on ${SERVED}, predecessor-registry on ${HELD_REGISTRY}`);
|
||||||
|
|
||||||
|
// The predecessor's control, which the operator stops before adopting (the record's words).
|
||||||
|
const unit = join(tmpdir(), `mesh-lab-stand-in-${process.pid}`);
|
||||||
|
writeFileSync(unit, STAND_IN);
|
||||||
|
await push(instanceId, CONTROL, unit, "/etc/systemd/system/predecessor-sync.service");
|
||||||
|
await must(CONTROL, `systemctl daemon-reload && systemctl start predecessor-sync && sleep 12 && systemctl stop predecessor-sync`);
|
||||||
|
said.push(` stand-in predecessor-sync rewrote ${SERVICE_FILE}, then the operator stopped it`);
|
||||||
|
|
||||||
|
// What the machine was, recorded ON the machine so a restored warm instance still has it.
|
||||||
|
await must(CONTROL, [
|
||||||
|
`mkdir -p ${BEFORE}`,
|
||||||
|
`sha256sum ${SERVICE_FILE} | cut -d' ' -f1 > ${BEFORE}/file.sha256`,
|
||||||
|
`cp ${SERVICE_FILE} ${BEFORE}/file`,
|
||||||
|
`docker inspect -f '{{.Id}}' ${SERVICE} > ${BEFORE}/container.id`,
|
||||||
|
`docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY} > ${BEFORE}/nopolicy.id`,
|
||||||
|
`ufw show added > ${BEFORE}/ufw-added.txt`,
|
||||||
|
].join(" && "));
|
||||||
|
await until(`the predecessor's ${SERVICE} answers the joiner`, 60, async () =>
|
||||||
|
(await on(JOINER, `curl -s --max-time 3 http://${ANCHOR}:${SERVED}/`)).out.includes("hello from the predecessor") ? true : null,
|
||||||
|
() => "no answer");
|
||||||
|
said.push((await must(CONTROL, `ufw status verbose`)).trim());
|
||||||
|
said.push((await must(CONTROL, `docker ps --format '{{.Names}}\t{{.Ports}}'`)).trim());
|
||||||
|
return said.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- steps, recorded rather than thrown --------------------------------------------------------
|
||||||
|
|
||||||
|
interface Step { code: string; title: string; ok: boolean; why: string; said: string; seconds: number; warm?: boolean }
|
||||||
|
const steps = new Map<string, Step>();
|
||||||
|
|
||||||
|
async function step(code: string, needs: string[], fn: () => Promise<string>): Promise<void> {
|
||||||
|
const title = TITLE[code]!;
|
||||||
|
const missing = needs.filter((n) => !steps.get(n)?.ok);
|
||||||
|
if (missing.length) {
|
||||||
|
steps.set(code, { code, title, ok: false, seconds: 0, said: "",
|
||||||
|
why: `not attempted — ${missing.join(", ")} did not succeed` });
|
||||||
|
console.log(`[${code}] SKIP ${title}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.log(`\n[${code}] ---- ${title} ----`);
|
||||||
|
const began = Date.now();
|
||||||
|
const took = () => Math.round((Date.now() - began) / 1000);
|
||||||
|
try {
|
||||||
|
const said = await fn();
|
||||||
|
steps.set(code, { code, title, ok: true, why: "", said, seconds: took() });
|
||||||
|
console.log(`${said}\n[${code}] PASS ${title} (${took()}s)`);
|
||||||
|
} catch (err) {
|
||||||
|
const why = (err as Error).message;
|
||||||
|
steps.set(code, { code, title, ok: false, why, said: "", seconds: took() });
|
||||||
|
console.log(`[${code}] FAIL ${title} (${took()}s)\n${why.split("\n").slice(0, 40).join("\n")}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The plan, in the record's order. Codes are stable; titles may be reworded. */
|
||||||
|
const TITLE: Record<string, string> = {
|
||||||
|
P0: "the machine is prepared the way the predecessor leaves one",
|
||||||
|
F0: "a converged genesis on a FRESH machine is not refused — its own resolver does not make it in use (ADR 0101)",
|
||||||
|
A1: "a converged genesis refuses the machine in use, naming every container and listener it counted",
|
||||||
|
A2: "an adopted genesis refuses the registry's held port, naming what holds it",
|
||||||
|
A3: "given another registry port, the adopted foundation comes up — and stays on that port as modules",
|
||||||
|
B1: "nothing that serves changed: the service answers, its file and container are untouched, the firewall gained only the mesh's marked rules",
|
||||||
|
B2: "the store is unreachable from outside, before and after the found firewall reloads; the bus answers a machine not yet enrolled",
|
||||||
|
B4: "the guard lets the machine's own containers reach the store (with the found firewall admitting them)",
|
||||||
|
C1: "a second machine enrols through the found firewall and joins the private network",
|
||||||
|
B3: "the store is reachable over the private network",
|
||||||
|
C2: "after the found firewall reloads, the openings are there and the mesh still works",
|
||||||
|
C3: "after the machine reboots, the openings are there and the mesh still works",
|
||||||
|
D1: "assigning prepares: the module holds the found container and file, and neither changes",
|
||||||
|
D2: "a predecessor still writing is caught: the held file's change is reported, and not reverted",
|
||||||
|
D3: "converging refuses while the service's module holds its found container",
|
||||||
|
D4: "taking cuts over: the found container and file are replaced, the original kept, the port reachable",
|
||||||
|
E1: "converging previews: the service's port, a published port no rule mentions, and the modules it takes",
|
||||||
|
E2: "the flip: the derived filter loaded, the found firewall disabled with its configuration on disk, the declared port open and the undeclared closed",
|
||||||
|
E3: "returned to adopted: the found firewall enabled again, the derived filter gone, the openings back",
|
||||||
|
F1: "unassigning takes the mesh's opening away and leaves the operator's own rule",
|
||||||
|
};
|
||||||
|
|
||||||
|
function stateOutcome(): void {
|
||||||
|
console.log(`\n================ ADOPTION: WHAT WAS ESTABLISHED ================`);
|
||||||
|
let established = 0;
|
||||||
|
for (const code of Object.keys(TITLE)) {
|
||||||
|
const s = steps.get(code);
|
||||||
|
const mark = !s ? "NEVER" : s.warm ? "WARM" : s.ok ? "PASS" : s.why.startsWith("not attempted") ? "SKIP" : "FAIL";
|
||||||
|
if (s?.ok) established++;
|
||||||
|
console.log(` ${code.padEnd(3)} ${mark.padEnd(5)} ${TITLE[code]}${s?.seconds ? ` (${s.seconds}s)` : ""}`);
|
||||||
|
}
|
||||||
|
console.log(` ${established}/${Object.keys(TITLE).length} established.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the run -----------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
if (skip) return;
|
||||||
|
console.log(`\n================ THE PLAN ================`);
|
||||||
|
for (const [code, title] of Object.entries(TITLE)) console.log(` ${code.padEnd(3)} ${title}`);
|
||||||
|
|
||||||
|
const verdict = WARM ? await ready(SCENARIO) : { use: "raise" as const, why: "not asked to be warm" };
|
||||||
|
let restored = false;
|
||||||
|
if (verdict.use === "restore") {
|
||||||
|
instanceId = verdict.instanceId;
|
||||||
|
const seconds = await returnTo(instanceId, (m) => console.log(`warm: ${m}`));
|
||||||
|
console.log(`WARM: restored ${instanceId} to the adopted foundation in ${seconds}s`);
|
||||||
|
restored = true;
|
||||||
|
for (const code of ["P0", "F0", "A1", "A2", "A3"]) {
|
||||||
|
steps.set(code, { code, title: TITLE[code]!, ok: true, warm: true, seconds: 0, why: "",
|
||||||
|
said: "restored from the warm snapshot — not re-run; only a fresh run proves it" });
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (WARM) console.log(`WARM: raising — ${verdict.why}`);
|
||||||
|
const bed = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
|
...(FIXED_ID ? { instanceId: FIXED_ID } : {}),
|
||||||
|
});
|
||||||
|
instanceId = bed.instanceId;
|
||||||
|
}
|
||||||
|
console.log(`INSTANCE ${instanceId}${KEEP ? " (KEEP — will be left standing)" : ""}`);
|
||||||
|
|
||||||
|
let heads: Record<string, string> = {};
|
||||||
|
const genesisOn = (node: string, o: Partial<GenesisOptions>): Promise<ReturnType<typeof genesis> extends Promise<infer R> ? R : never> =>
|
||||||
|
genesis({
|
||||||
|
instanceId, node, installer: installer as string, catalogDir,
|
||||||
|
bundleTemplate: foundationBundle(bundle, []),
|
||||||
|
registry: `${ANCHOR}:${HELD_REGISTRY}`,
|
||||||
|
source: forgeUrl("mesh-controller"), sourceRef: heads["mesh-controller"] ?? "",
|
||||||
|
toolsSource: forgeUrl("mesh-tools"), toolsRef: "lab",
|
||||||
|
catalogSource: forgeUrl("mesh-catalog"), catalogRef: "lab",
|
||||||
|
sdkSource: forgeUrl("mesh-sdk"), sdkRef: "lab",
|
||||||
|
site: "hosting",
|
||||||
|
// The service, so a machine that reboots comes back holding itself (the bed reboots one).
|
||||||
|
// F0 asks for a dry run on a machine that must stay fresh, and passes false for itself.
|
||||||
|
hostService: true,
|
||||||
|
...(binary ? { hostBinary: binary } : {}),
|
||||||
|
log: (m) => console.log(m),
|
||||||
|
...o,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!restored) {
|
||||||
|
await step("P0", [], async () => {
|
||||||
|
heads = await raiseForge();
|
||||||
|
const said = [` forge git://${FORGE}/ serving ${Object.entries(heads).map(([n, h]) => `${n}@${h.slice(0, 8)}`).join(", ")}`];
|
||||||
|
said.push(await preparePredecessor());
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
// ADR 0101: the joiner is a freshly installed machine — nothing but its operating system, a
|
||||||
|
// container runtime and the host binary. A converged genesis there must not be refused. Asked
|
||||||
|
// as a dry run: the question is the preflight's, and a real raise would make it a second mesh.
|
||||||
|
await step("F0", ["P0"], async () => {
|
||||||
|
const ran = await genesisOn(JOINER, { flags: ["--dry-run"], attempts: 1, verify: false, hostService: false });
|
||||||
|
assert.doesNotMatch(ran.said, /this machine is in use/,
|
||||||
|
`a converged genesis refused a fresh machine as in use:\n${ran.said}`);
|
||||||
|
assert.match(ran.said, /in use\s+no: no container runs and nothing listens beyond ssh/,
|
||||||
|
`the installer never said the fresh machine is not in use:\n${ran.said}`);
|
||||||
|
const listening = (await must(JOINER, `ss -Hltunp`)).trim();
|
||||||
|
return ` in use no — the installer went on (${ran.ok ? "dry run finished" : `dry run stopped later, at ${ran.step}`})\n` +
|
||||||
|
` what listens on the fresh machine:\n${listening.split("\n").map((l) => ` ${l}`).join("\n")}`;
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("A1", ["P0"], async () => {
|
||||||
|
const ran = await genesisOn(CONTROL, { attempts: 1, verify: false });
|
||||||
|
assert.ok(!ran.ok, `a converged genesis went ahead on a machine in use:\n${ran.said}`);
|
||||||
|
assert.match(ran.step, /preflight/, `it was refused, but not before changing anything (at ${ran.step}):\n${ran.said}`);
|
||||||
|
for (const want of [/container hello-web/, /container predecessor-registry/,
|
||||||
|
new RegExp(`tcp \\S+:${SERVED} by`), new RegExp(`tcp \\S+:${HELD_REGISTRY} by`)]) {
|
||||||
|
assert.match(ran.said, want, `the refusal does not name ${want}:\n${ran.said}`);
|
||||||
|
}
|
||||||
|
assert.match(ran.said, /--adopted/, `the refusal does not say how to raise it adopted`);
|
||||||
|
// And nothing was changed: the predecessor as it was, no table of the mesh's.
|
||||||
|
const ps = await must(CONTROL, `docker ps --format '{{.Names}}'`);
|
||||||
|
assert.deepEqual(ps.trim().split("\n").sort(), [SERVICE, NO_POLICY, "predecessor-registry"].sort(), `containers changed:\n${ps}`);
|
||||||
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
|
||||||
|
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
|
||||||
|
return ran.said.split("\n").filter((l) => /in use|^\s+- /.test(l)).join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("A2", ["A1"], async () => {
|
||||||
|
const ran = await genesisOn(CONTROL, { adopted: true, attempts: 1, verify: false });
|
||||||
|
assert.ok(!ran.ok, `an adopted genesis went ahead with the registry's port held:\n${ran.said}`);
|
||||||
|
assert.match(ran.said, new RegExp(`registry's port tcp/${HELD_REGISTRY} is held by [^\\n]*predecessor-registry`),
|
||||||
|
`the refusal does not name what holds the registry's port:\n${ran.said.split("\n").slice(-15).join("\n")}`);
|
||||||
|
assert.match(ran.said, /--registry-port/, `the refusal does not say which flag gives another port`);
|
||||||
|
const id = (await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim();
|
||||||
|
assert.equal(id, (await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the predecessor's container was replaced`);
|
||||||
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/);
|
||||||
|
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `a mesh table was loaded`);
|
||||||
|
return ` refused at ${ran.step}\n` + ran.said.split("\n").filter((l) => /held by|port|adopted/.test(l)).slice(-8).join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("A3", ["A2"], async () => {
|
||||||
|
const ran = await genesisOn(CONTROL, { adopted: true, registry: `${ANCHOR}:${REGISTRY_PORT}` });
|
||||||
|
if (!ran.ok) throw new Error(`${ran.step}: ${ran.why}\n\n${ran.report.join("\n")}`);
|
||||||
|
await settled();
|
||||||
|
const said = [ran.report.join("\n")];
|
||||||
|
const bindings = await must(CONTROL, `docker inspect -f '{{json .HostConfig.PortBindings}}' mesh-registry`);
|
||||||
|
assert.match(bindings, new RegExp(`"HostPort":"${REGISTRY_PORT}"`), `the registry is not on ${REGISTRY_PORT}: ${bindings}`);
|
||||||
|
assert.match(await must(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' mesh-registry`), /\S/,
|
||||||
|
`mesh-registry is not the host's: the foundation was not adopted as a module`);
|
||||||
|
assert.match(await mesh(`module list`), /^distribution\b/m, `the registry is not a module the mesh holds`);
|
||||||
|
// The node's own port, in what the mesh would send it — not the catalogue's default.
|
||||||
|
const plan = await mesh(`plan ${CONTROL} --json`);
|
||||||
|
assert.match(plan, new RegExp(`"${REGISTRY_PORT}:5000"`), `the registry's module does not publish ${REGISTRY_PORT}`);
|
||||||
|
assert.doesNotMatch(plan, /"5000:5000"/, `the plan still publishes the catalogue's 5000`);
|
||||||
|
said.push(` registry on ${REGISTRY_PORT}, as the module the mesh holds: ${bindings.trim()}`);
|
||||||
|
const show = await nodeShow(CONTROL);
|
||||||
|
assert.match(show, /mode\s+adopted since/, `the anchor is not reported adopted:\n${show}`);
|
||||||
|
assert.ok(!(await on(CONTROL, `nft list table inet mesh`)).ok, `the foundation's dropping table was loaded on an adopted node`);
|
||||||
|
const guard = await must(CONTROL, `nft list table inet mesh_guard`);
|
||||||
|
// The guard's port set is the controller's to derive; what the record fixes is that it refuses
|
||||||
|
// the store's port from outside and holds nothing but refusals.
|
||||||
|
assert.match(guard, new RegExp(`dport (\\{[^}]*\\b${STORE_PORT}\\b[^}]*\\}|${STORE_PORT}) drop`), `the guard does not refuse the store's port:\n${guard}`);
|
||||||
|
assert.doesNotMatch(guard, /accept\s*$/m, `the guard holds an accept:\n${guard}`);
|
||||||
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force`);
|
||||||
|
assert.match(await must(CONTROL, `curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${HELD_REGISTRY}/v2/`), /200/,
|
||||||
|
`the predecessor's registry stopped answering`);
|
||||||
|
said.push(show.trim(), guard.trim());
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
if (WARM && steps.get("A3")?.ok) {
|
||||||
|
await keep(SCENARIO, instanceId);
|
||||||
|
console.log(`WARM: kept ${instanceId} at the adopted foundation`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- nothing that serves changed -------------------------------------------------------------
|
||||||
|
await step("B1", ["A3"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
const want = await must(CONTROL, `cat ${BEFORE}/file`);
|
||||||
|
let page = "";
|
||||||
|
await until(`the service answers the joiner as it did`, 120, async () => {
|
||||||
|
page = (await on(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`)).out;
|
||||||
|
return page === want ? true : null;
|
||||||
|
}, () => page);
|
||||||
|
said.push(` ${SERVICE} answers the joiner on ${SERVED} with the predecessor's page`);
|
||||||
|
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
|
||||||
|
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `${SERVICE_FILE} changed`);
|
||||||
|
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
||||||
|
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the ${SERVICE} container was replaced`);
|
||||||
|
said.push(` file, container byte for byte / the same id as before the mesh`);
|
||||||
|
const was = (await must(CONTROL, `cat ${BEFORE}/ufw-added.txt`)).split("\n").map((l) => l.trim()).filter((l) => l.startsWith("ufw "));
|
||||||
|
const now = await ufwAdded();
|
||||||
|
const lost = was.filter((r) => !now.includes(r));
|
||||||
|
const added = now.filter((r) => !was.includes(r));
|
||||||
|
assert.deepEqual(lost, [], `the found firewall lost rules:\n${lost.join("\n")}`);
|
||||||
|
const unmarked = added.filter((r) => !marked(r));
|
||||||
|
assert.deepEqual(unmarked, [], `the found firewall gained rules not marked as the mesh's:\n${unmarked.join("\n")}`);
|
||||||
|
assert.ok(added.length > 0, `the mesh opened nothing through the found firewall`);
|
||||||
|
// ADR 0102: the runtime's file is written into, never over, and the runtime is reloaded.
|
||||||
|
const daemon = JSON.parse(await must(CONTROL, `cat /etc/docker/daemon.json`)) as
|
||||||
|
{ "log-opts"?: Record<string, string>; "insecure-registries"?: string[] };
|
||||||
|
assert.equal(daemon["log-opts"]?.["max-size"], "7m", `the machine's own runtime setting was replaced: ${JSON.stringify(daemon)}`);
|
||||||
|
assert.ok((daemon["insecure-registries"] ?? []).some((r) => r.includes(":")),
|
||||||
|
`the mesh's registry trust is not in the runtime's file: ${JSON.stringify(daemon)}`);
|
||||||
|
assert.ok((daemon["insecure-registries"] ?? []).length > 1,
|
||||||
|
`the machine's own registries were replaced rather than added to: ${JSON.stringify(daemon)}`);
|
||||||
|
const stillUp = (await must(CONTROL, `docker inspect -f '{{.State.Running}} {{.Id}}' ${NO_POLICY}`)).trim();
|
||||||
|
assert.match(stillUp, /^true /, `the container with no restart policy is not running: ${stillUp}`);
|
||||||
|
assert.equal(stillUp, (await must(CONTROL, `cat ${BEFORE}/nopolicy.id`)).trim(),
|
||||||
|
`the container with no restart policy was restarted or replaced`);
|
||||||
|
said.push(` runtime file the machine's log-opts kept, the mesh's registry added; ${NO_POLICY} still up`);
|
||||||
|
said.push(` firewall ${was.length} rule(s) kept, ${added.length} added, every one marked:`, ...added.map((r) => ` ${r}`));
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("B2", ["A3"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers a machine off the private network`);
|
||||||
|
await must(CONTROL, `ufw reload`);
|
||||||
|
await sleep(3_000);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the found firewall reloaded`);
|
||||||
|
said.push(` outsider -> ${STORE_PORT} refused, before and after \`ufw reload\``);
|
||||||
|
assert.ok(await connects(OUTSIDER, ANCHOR, BUS_PORT), `the bus does not answer a machine that has not enrolled`);
|
||||||
|
said.push(` outsider -> ${BUS_PORT} the bus answers`);
|
||||||
|
|
||||||
|
// **What the guard is for** (ADR 0100, 0103): the store must be unreachable from outside
|
||||||
|
// *whatever the found firewall does*. With ufw admitting both ports, only the guard is left
|
||||||
|
// between the outsider and the store — and with the guard gone they are reachable, which is
|
||||||
|
// what makes this a test of the guard rather than of ufw.
|
||||||
|
const admit = [STORE_PORT, AMQP_PORT].map((p) =>
|
||||||
|
`ufw route allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`);
|
||||||
|
try {
|
||||||
|
await must(CONTROL, admit.join(" && "));
|
||||||
|
await sleep(2_000);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)),
|
||||||
|
`the store answers from outside once the found firewall admits it — the guard refuses nothing`);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, AMQP_PORT)),
|
||||||
|
`the broker's plaintext port answers from outside once the found firewall admits it`);
|
||||||
|
said.push(` outsider -> ${STORE_PORT}, ${AMQP_PORT} still refused with the found firewall admitting both — the guard's own refusal`);
|
||||||
|
|
||||||
|
// The positive control: without the guard, both answer. Anything else would mean the probe
|
||||||
|
// could not have failed.
|
||||||
|
await must(CONTROL, `nft delete table inet mesh_guard`);
|
||||||
|
await sleep(2_000);
|
||||||
|
const openNow = (await connects(OUTSIDER, ANCHOR, STORE_PORT)) || (await connects(OUTSIDER, ANCHOR, AMQP_PORT));
|
||||||
|
await must(CONTROL, `systemctl reload mesh-guard.service || systemctl restart mesh-guard.service`);
|
||||||
|
await sleep(2_000);
|
||||||
|
assert.ok(openNow, `neither port answered with the guard deleted, so the guard is not what refuses them`);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store stayed open after the guard was loaded again`);
|
||||||
|
said.push(` guard deleted both answered; loaded again, both refused`);
|
||||||
|
} finally {
|
||||||
|
await on(CONTROL, [STORE_PORT, AMQP_PORT].map((p) =>
|
||||||
|
`ufw route delete allow proto tcp to any port ${p} comment 'bed-probe-only ${p}'`).join("; "));
|
||||||
|
}
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
// Its own step: it asks something different of the found firewall — a container on the machine
|
||||||
|
// reaches a published port through the runtime's proxy, on the incoming path, not the forwarded.
|
||||||
|
await step("B4", ["A3"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
// What this asks is the guard's promise: it never refuses the machine's own containers. The
|
||||||
|
// found firewall stays in force (ADR 0100) and denies inbound by default, and a container on
|
||||||
|
// the store's own network reaches its published port through the runtime's proxy — inbound,
|
||||||
|
// not forwarded — so the operator's firewall has to admit the container interface for any
|
||||||
|
// container to get there, on an adopted node as on a converged one. The probe admits it for
|
||||||
|
// itself alone, and takes the rule away again.
|
||||||
|
await must(CONTROL, `ufw allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`);
|
||||||
|
let fromContainer: { ok: boolean; out: string };
|
||||||
|
try {
|
||||||
|
fromContainer = await on(CONTROL, `docker run --rm ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`, 180_000);
|
||||||
|
} finally {
|
||||||
|
await on(CONTROL, `ufw delete allow in on docker0 to any port ${STORE_PORT} proto tcp comment bed-probe-only`);
|
||||||
|
}
|
||||||
|
if (!fromContainer.ok) {
|
||||||
|
// Evidence, so the cause can be read from this run rather than guessed at the next one.
|
||||||
|
const evidence = await on(CONTROL, [
|
||||||
|
`echo '--- published'; docker ps --format '{{.Names}} {{.Ports}}' | grep -i ${STORE_PORT}`,
|
||||||
|
`echo '--- from the host'; nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`,
|
||||||
|
`echo '--- from the host network'; docker run --rm --network host ${ALPINE} nc -zv -w 3 ${ANCHOR} ${STORE_PORT} 2>&1`,
|
||||||
|
`echo '--- iptables FORWARD, DOCKER-USER, isolation'; iptables -S FORWARD; iptables -S DOCKER-USER; iptables -S | grep -i isolation`,
|
||||||
|
`echo '--- the guard'; nft list table inet mesh_guard`,
|
||||||
|
`echo '--- nat for the port'; iptables -t nat -S | grep ${STORE_PORT}`,
|
||||||
|
].join("; "), 120_000);
|
||||||
|
assert.fail(`a container on the node cannot reach the store:\n${fromContainer.out}\n${evidence.out}`);
|
||||||
|
}
|
||||||
|
said.push(` container -> ${STORE_PORT} reachable from a container on the node itself`);
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- the mesh works through the found firewall -----------------------------------------------
|
||||||
|
let anchorOnMesh = "";
|
||||||
|
await step("C1", ["B2"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
await mesh(`node add ${JOINER}`);
|
||||||
|
const token = tokenFrom(await mesh(`token issue --node ${JOINER}`));
|
||||||
|
const out = await must(JOINER, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
|
||||||
|
assert.match(out, new RegExp(`enrolled as ${JOINER}`), out);
|
||||||
|
await must(JOINER, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||||
|
said.push(` ${JOINER} enrolled over the bus, through the anchor's ufw`);
|
||||||
|
await mesh(`overlay place ${JOINER} --site hosting`);
|
||||||
|
await mesh(`assign ${JOINER} ${NETWORK_MODULE}`);
|
||||||
|
await pushAndSettle(JOINER);
|
||||||
|
// The hub's peer list changed: the anchor has to be sent it too.
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
anchorOnMesh = await meshAddressOf(CONTROL);
|
||||||
|
await until(`the joiner reaches the anchor over mesh0`, 180, async () =>
|
||||||
|
(await on(JOINER, `ping -c1 -W2 ${anchorOnMesh}`)).ok ? true : null,
|
||||||
|
() => "no ping reply");
|
||||||
|
said.push(` private network the joiner reaches the anchor at ${anchorOnMesh}`);
|
||||||
|
said.push((await must(CONTROL, `wg show mesh0 latest-handshakes`)).trim());
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("B3", ["C1"], async () => {
|
||||||
|
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store does not answer over the private network`);
|
||||||
|
return ` joiner -> ${anchorOnMesh}:${STORE_PORT} reachable over mesh0`;
|
||||||
|
});
|
||||||
|
|
||||||
|
/** The mesh's own rules in the found firewall. */
|
||||||
|
const openings = async (): Promise<string[]> => (await ufwAdded()).filter(marked);
|
||||||
|
const assertOpenings = (rules: string[]) => {
|
||||||
|
const text = rules.join("\n");
|
||||||
|
// By the opening's id, which names the machine's port: a forwarded rule names the CONTAINER's
|
||||||
|
// port (ufw's route rules match after the runtime's translation), so the text may say another.
|
||||||
|
for (const port of [BUS_PORT, REGISTRY_PORT, HUB_PORT, STORE_PORT]) {
|
||||||
|
assert.match(text, new RegExp(`opening-(tcp|udp)-${port}-`), `no opening for ${port} among the mesh's rules:\n${text}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
await step("C2", ["B3"], async () => {
|
||||||
|
const before = await openings();
|
||||||
|
assertOpenings(before);
|
||||||
|
await must(CONTROL, `ufw reload`);
|
||||||
|
await sleep(3_000);
|
||||||
|
const after = await openings();
|
||||||
|
assert.deepEqual(after.sort(), before.sort(), `the openings changed across a reload`);
|
||||||
|
assert.ok(await connects(JOINER, anchorOnMesh, STORE_PORT), `the store stopped answering over mesh0 after the reload`);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reload`);
|
||||||
|
await pushAndSettle(JOINER);
|
||||||
|
return ` after ufw reload ${after.length} opening(s) in place; the joiner reaches the store over mesh0 and takes a push\n` +
|
||||||
|
after.map((r) => ` ${r}`).join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("C3", ["C2"], async () => {
|
||||||
|
const before = await openings();
|
||||||
|
await restartMachine(CONTROL);
|
||||||
|
await until(`the control plane answers after the reboot`, 300, async () =>
|
||||||
|
(await meshSays(`status`)).ok ? true : null, () => "no answer");
|
||||||
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `the found firewall is not in force after the reboot`);
|
||||||
|
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard did not come back`);
|
||||||
|
const after = await until(`the openings are there again`, 420, async () => {
|
||||||
|
const now = await openings();
|
||||||
|
return before.every((r) => now.includes(r)) ? now : null;
|
||||||
|
}, () => "not all openings");
|
||||||
|
assertOpenings(after);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside after the reboot`);
|
||||||
|
await until(`the joiner reaches the store over mesh0 again`, 300, async () =>
|
||||||
|
(await connects(JOINER, anchorOnMesh, STORE_PORT)) ? true : null, () => "no connection");
|
||||||
|
await pushAndSettle(JOINER);
|
||||||
|
const page = await must(JOINER, `curl -s --max-time 5 http://${ANCHOR}:${SERVED}/`);
|
||||||
|
assert.match(page, /hello from the predecessor/, `the predecessor's service did not come back: ${page}`);
|
||||||
|
return ` after a reboot ufw active, the guard loaded, ${after.length} opening(s); the joiner reaches the store and takes a push`;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- assigning prepares, taking cuts over ----------------------------------------------------
|
||||||
|
let kept = "";
|
||||||
|
await step("D1", ["B1"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
// The catalogue's module, read from the catalogue, with the route requirement taken off: the
|
||||||
|
// route needs a proxy module and a public name, and neither is what adoption is about.
|
||||||
|
const manifest = JSON.parse(catalogueModule(SERVICE, [])) as Record<string, unknown>;
|
||||||
|
delete manifest["requires"]; delete manifest["contributes"]; delete manifest["binds"];
|
||||||
|
await registerModule(SERVICE, JSON.stringify(manifest));
|
||||||
|
await mesh(`assign ${CONTROL} ${SERVICE}`);
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
const show = await until(`the anchor reports holding ${SERVICE}'s container and file`, 120, async () => {
|
||||||
|
const s = await nodeShow(CONTROL);
|
||||||
|
return /holds container\s+hello-web\b/.test(s) && /holds file\s+\/var\/lib\/hello-web\/index\.html/.test(s) ? s : null;
|
||||||
|
}, () => "");
|
||||||
|
kept = show.match(/holds file\s+\/var\/lib\/hello-web\/index\.html[^\n]*\n\s*original kept at (\S+)/)?.[1] ?? "";
|
||||||
|
assert.ok(kept, `the held file's original is not reported kept:\n${show}`);
|
||||||
|
assert.equal((await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`)).trim(),
|
||||||
|
(await must(CONTROL, `cat ${BEFORE}/file.sha256`)).trim(), `assigning changed ${SERVICE_FILE}`);
|
||||||
|
assert.equal((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
||||||
|
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `assigning replaced the ${SERVICE} container`);
|
||||||
|
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the kept original is not the file as found`);
|
||||||
|
said.push(show.trim());
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("D2", ["D1"], async () => {
|
||||||
|
await must(CONTROL, `systemctl start predecessor-sync`);
|
||||||
|
try {
|
||||||
|
await sleep(15_000);
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
const show = await until(`the anchor reports the held file changed`, 120, async () => {
|
||||||
|
const s = await nodeShow(CONTROL);
|
||||||
|
return /hello-web\/index\.html[^\n]*REWRITTEN/i.test(s) ? s : null;
|
||||||
|
}, () => "");
|
||||||
|
// Stop the writer first, then hash: while it rewrites every ten seconds, a file the host
|
||||||
|
// reverted in between would still read as the predecessor's a moment later.
|
||||||
|
await must(CONTROL, `systemctl stop predecessor-sync`);
|
||||||
|
const was = await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`);
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
await sleep(5_000);
|
||||||
|
const now = await must(CONTROL, `cat ${SERVICE_FILE}`);
|
||||||
|
assert.match(now, /synced \d+/, `the host reverted what the predecessor wrote:\n${now}`);
|
||||||
|
assert.equal(await must(CONTROL, `sha256sum ${SERVICE_FILE} | cut -d' ' -f1`), was,
|
||||||
|
`the held file changed under a push after the predecessor stopped writing`);
|
||||||
|
return show.trim();
|
||||||
|
} finally {
|
||||||
|
await on(CONTROL, `systemctl stop predecessor-sync`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("D3", ["D1"], async () => {
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
const r = await meshSays(`converge ${CONTROL}`);
|
||||||
|
assert.ok(!r.ok, `converge went ahead while ${SERVICE} holds its found container:\n${r.out}`);
|
||||||
|
assert.match(r.out, new RegExp(`hello-web holds the found container hello-web`), `the refusal does not name the held container:\n${r.out}`);
|
||||||
|
assert.match(r.out, new RegExp(`take ${CONTROL} hello-web`), `the refusal does not say what to do:\n${r.out}`);
|
||||||
|
return r.out.trim();
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("D4", ["D1"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
said.push((await mesh(`take ${CONTROL} ${SERVICE}`)).trim());
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
const label = await until(`the ${SERVICE} container is the mesh's`, 180, async () => {
|
||||||
|
const l = (await on(CONTROL, `docker inspect -f '{{index .Config.Labels "mesh-host.spec"}}' ${SERVICE}`)).out.trim();
|
||||||
|
return l && l !== "<no value>" ? l : null;
|
||||||
|
}, () => "");
|
||||||
|
assert.notEqual((await must(CONTROL, `docker inspect -f '{{.Id}}' ${SERVICE}`)).trim(),
|
||||||
|
(await must(CONTROL, `cat ${BEFORE}/container.id`)).trim(), `the found container was not replaced`);
|
||||||
|
const catalogue = (JSON.parse(catalogueModule(SERVICE, [])) as { resources: { id: string; content?: string }[] })
|
||||||
|
.resources.find((r) => r.id === "page")?.content ?? "";
|
||||||
|
assert.equal(await must(CONTROL, `cat ${SERVICE_FILE}`), catalogue, `the found file was not replaced with the module's`);
|
||||||
|
assert.equal(await must(CONTROL, `cat ${kept}`), await must(CONTROL, `cat ${BEFORE}/file`), `the original was not kept`);
|
||||||
|
said.push(` replaced container (spec ${label.slice(0, 12)}…) and ${SERVICE_FILE}; original still at ${kept}`);
|
||||||
|
// Either the mesh opened the port, or the predecessor's own rule already admits it — then the
|
||||||
|
// mesh adds nothing and will remove nothing (ADR 0103), and the operator's rule stays theirs.
|
||||||
|
const opened = (await openings()).filter((r) => new RegExp(`opening-tcp-${SERVED}-`).test(r));
|
||||||
|
const operators = (await ufwAdded()).filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r));
|
||||||
|
assert.ok(opened.length > 0 || operators.length > 0,
|
||||||
|
`no opening for ${SERVED} once ${SERVICE} was taken, and no rule of the operator's admits it:\n${(await ufwAdded()).join("\n")}`);
|
||||||
|
assert.ok(operators.length > 0, `the operator's own rule for ${SERVED} is gone:\n${(await ufwAdded()).join("\n")}`);
|
||||||
|
said.push(...opened.map((r) => ` opened ${r}`));
|
||||||
|
if (opened.length === 0) said.push(` opening ${SERVED} satisfied by the operator's own rule: ${operators.join(" | ")}`);
|
||||||
|
const page = await until(`the taken ${SERVICE} answers over the private network`, 120, async () => {
|
||||||
|
const p = await on(JOINER, `curl -s --max-time 3 http://${anchorOnMesh}:${SERVED}/`);
|
||||||
|
return p.ok && p.out === catalogue ? p.out : null;
|
||||||
|
}, () => "");
|
||||||
|
said.push(` joiner -> ${SERVED} ${page.trim()}`);
|
||||||
|
const show = await nodeShow(CONTROL);
|
||||||
|
assert.doesNotMatch(show, /holds (container|file)\s+\S*hello-web/, `the anchor still holds what was taken:\n${show}`);
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- converging previews, then changes -------------------------------------------------------
|
||||||
|
await step("E1", ["D4"], async () => {
|
||||||
|
if (!/^nftables\b/m.test(await mesh(`module list`))) {
|
||||||
|
await registerModule(FILTER_MODULE, JSON.stringify(JSON.parse(catalogueModule(FILTER_MODULE, []))));
|
||||||
|
}
|
||||||
|
// What the flip will close must be reachable now, or its closing proves nothing.
|
||||||
|
assert.ok(await connects(JOINER, anchorOnMesh, HELD_REGISTRY),
|
||||||
|
`the predecessor's published ${HELD_REGISTRY} is not reachable over the private network before the flip`);
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
const preview = await mesh(`converge ${CONTROL}`);
|
||||||
|
assert.match(preview, new RegExp(`tcp/${SERVED}\\b[^\\n]*declared by hello-web`), `the preview does not name the service's port as declared:\n${preview}`);
|
||||||
|
assert.match(preview, new RegExp(`tcp/${HELD_REGISTRY}\\b[^\\n]*published[^\\n]*WILL CLOSE`), `the preview does not name the published ${HELD_REGISTRY} as closing:\n${preview}`);
|
||||||
|
assert.match(preview, /the flip takes:\n(\s{4}\S+\n?)+/, `the preview names no module the flip takes:\n${preview}`);
|
||||||
|
assert.match(preview, new RegExp(`\\n\\s{4}${NETWORK_MODULE}\\b`), `the preview does not say the flip takes ${NETWORK_MODULE}:\n${preview}`);
|
||||||
|
assert.match(preview, /Nothing has changed/, preview);
|
||||||
|
assert.match(await must(CONTROL, `ufw status`), /Status: active/, `previewing changed the firewall`);
|
||||||
|
return preview.trim();
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("E2", ["E1"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
// The flip as the preview says to make it — whatever the preview binds `--yes` to.
|
||||||
|
const preview = await mesh(`converge ${CONTROL}`);
|
||||||
|
const flip = preview.match(new RegExp(`\`(converge ${CONTROL} --yes[^\`]*)\``))?.[1];
|
||||||
|
assert.ok(flip, `the preview does not say how to make the flip:\n${preview}`);
|
||||||
|
said.push((await mesh(flip, 300_000)).trim().split("\n").slice(-3).join("\n"));
|
||||||
|
await settled();
|
||||||
|
const table = await until(`the derived filter is loaded`, 300, async () => {
|
||||||
|
const t = await on(CONTROL, `nft list table inet mesh`);
|
||||||
|
return t.ok && /policy drop/.test(t.out) ? t.out : null;
|
||||||
|
}, () => "");
|
||||||
|
const status = await until(`the found firewall is disabled`, 180, async () => {
|
||||||
|
const s = (await on(CONTROL, `ufw status`)).out;
|
||||||
|
return /Status: inactive/.test(s) ? s : null;
|
||||||
|
}, () => "");
|
||||||
|
assert.ok((await on(CONTROL, `test -s /etc/ufw/user.rules && grep -q 'BEGIN UFW AND DOCKER' /etc/ufw/after.rules`)).ok,
|
||||||
|
`the found firewall's configuration is not on disk any more`);
|
||||||
|
assert.match(await nodeShow(CONTROL), /mode\s+converged/, `the anchor is not reported converged`);
|
||||||
|
said.push(` ufw ${status.trim()} — /etc/ufw/user.rules and after.rules still on disk`);
|
||||||
|
await until(`the declared ${SERVED} answers over the private network`, 120, async () =>
|
||||||
|
(await connects(JOINER, anchorOnMesh, SERVED)) ? true : null, () => "");
|
||||||
|
assert.ok(!(await connects(JOINER, anchorOnMesh, HELD_REGISTRY)), `the undeclared ${HELD_REGISTRY} is still open`);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once converged`);
|
||||||
|
said.push(` ports ${SERVED} open over the private network; ${HELD_REGISTRY} closed; the store still closed from outside`);
|
||||||
|
said.push(table.split("\n").slice(0, 30).join("\n"));
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
await step("E3", ["E2"], async () => {
|
||||||
|
const said = (await mesh(`adopt ${CONTROL}`, 300_000)).trim();
|
||||||
|
await settled();
|
||||||
|
await until(`the found firewall is enabled again`, 180, async () =>
|
||||||
|
/Status: active/.test((await on(CONTROL, `ufw status`)).out) ? true : null, () => "");
|
||||||
|
await until(`the derived filter is gone`, 180, async () =>
|
||||||
|
!(await on(CONTROL, `nft list table inet mesh`)).ok ? true : null, () => "");
|
||||||
|
assert.match(await must(CONTROL, `nft list table inet mesh_guard`), /drop/, `the guard is not restored`);
|
||||||
|
assertOpenings(await until(`the openings are back`, 180, async () => {
|
||||||
|
const o = await openings();
|
||||||
|
return o.length ? o : null;
|
||||||
|
}, () => ""));
|
||||||
|
assert.match(await nodeShow(CONTROL), /mode\s+adopted since/, `the anchor is not reported adopted`);
|
||||||
|
assert.ok(!(await connects(OUTSIDER, ANCHOR, STORE_PORT)), `the store answers from outside once adopted again`);
|
||||||
|
return `${said}\n ufw active, table inet mesh gone, the guard and the openings back`;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- what the mesh added, it takes back; what it found, it leaves ---------------------------
|
||||||
|
await step("F1", ["E3"], async () => {
|
||||||
|
const said: string[] = [];
|
||||||
|
const before = await ufwAdded();
|
||||||
|
const operators = before.filter((r) => !marked(r) && new RegExp(`\\b${SERVED}\\b`).test(r));
|
||||||
|
assert.ok(operators.length > 0, `the operator's own rules for ${SERVED} are already gone:\n${before.join("\n")}`);
|
||||||
|
await mesh(`unassign ${CONTROL} ${SERVICE}`);
|
||||||
|
await pushAndSettle(CONTROL);
|
||||||
|
let lastRules: string[] = before;
|
||||||
|
const after = await until(`the mesh's own rules for ${SERVED} are gone`, 180, async () => {
|
||||||
|
const rules = await ufwAdded();
|
||||||
|
lastRules = rules;
|
||||||
|
return rules.some((r) => marked(r) && new RegExp(`opening-tcp-${SERVED}-`).test(r)) ? null : rules;
|
||||||
|
}, () => lastRules.join("\n"));
|
||||||
|
for (const rule of operators) {
|
||||||
|
assert.ok(after.includes(rule), `the operator's own rule went with the mesh's opening: ${rule}\n${after.join("\n")}`);
|
||||||
|
}
|
||||||
|
said.push(` kept ${operators.length} rule(s) of the operator's, unmarked, after the module was unassigned`);
|
||||||
|
said.push(...operators.map((r) => ` ${r}`));
|
||||||
|
return said.join("\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
stateOutcome();
|
||||||
|
}, { timeout: 10_800_000 });
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
if (KEEP || WARM) {
|
||||||
|
console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (${KEEP ? "MESH_LAB_KEEP" : "MESH_LAB_WARM"}).`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (instanceId) await destroy(instanceId);
|
||||||
|
await destroyAll(`${SCENARIO}-`);
|
||||||
|
}, { timeout: 900_000 });
|
||||||
|
|
||||||
|
for (const [code, title] of Object.entries(TITLE)) {
|
||||||
|
test(`${code} ${title}`, { skip, timeout: 60_000 }, () => {
|
||||||
|
const s = steps.get(code);
|
||||||
|
assert.ok(s?.ok, s ? `${s.why}` : `${code} never ran`);
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -49,7 +49,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -62,7 +62,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "anthropic-bed";
|
const SCENARIO = "anthropic-bed";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -194,8 +194,6 @@ after(async () => {
|
|||||||
test("model access refreshes on the manager node and delivers only the access token, never the refresh token", {
|
test("model access refreshes on the manager node and delivers only the access token, never the refresh token", {
|
||||||
skip, timeout: 1_500_000,
|
skip, timeout: 1_500_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
const managerImage = pinned("mesh-runtime-anthropic-manager");
|
|
||||||
const consumerImage = pinned("mesh-runtime-anthropic-consumer");
|
|
||||||
|
|
||||||
// --- the licence, and the manager as its holder ------------------------------------------------
|
// --- the licence, and the manager as its holder ------------------------------------------------
|
||||||
// The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token;
|
// The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token;
|
||||||
@@ -207,37 +205,15 @@ test("model access refreshes on the manager node and delivers only the access to
|
|||||||
await mesh(`licence use personal ${MACHINE} anthropic-manager`);
|
await mesh(`licence use personal ${MACHINE} anthropic-manager`);
|
||||||
|
|
||||||
// --- the manager module, deployed so the host delivers its bound facts --------------------------
|
// --- the manager module, deployed so the host delivers its bound facts --------------------------
|
||||||
// Inline manifest mirroring the committed module.json: model-access holder, refresh token bound as a
|
// The catalogue's own manifest (novox/hq 04-ISSUES/073): model-access holder, refresh token bound
|
||||||
// sealed secret, no node-key mount. The scheduled container installs as present state (ADR 0053);
|
// as a sealed secret, no node-key mount. The scheduled container installs as present state (ADR
|
||||||
// the test drives adopt/refresh directly for a deterministic flow rather than waiting on cron.
|
// 0053); the test drives adopt/refresh directly for a deterministic flow rather than waiting on
|
||||||
const managerManifest = JSON.stringify({
|
// cron. The one lab rewrite: the OAuth endpoints point at the stub this bed raises below.
|
||||||
module: "anthropic-manager",
|
const managerManifest = catalogueModule("anthropic-manager", held, {
|
||||||
version: "1",
|
env: { refresh: {
|
||||||
requires: ["model-access"],
|
MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token",
|
||||||
binds: { "model-access": "/var/lib/mesh/anthropic-manager/model.json" },
|
MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage",
|
||||||
secrets: { "model-access": "/var/lib/mesh/anthropic-manager/refresh-token" },
|
} },
|
||||||
"own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" },
|
|
||||||
emits: ["module.anthropic-manager.usage.read"],
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" },
|
|
||||||
{ id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh",
|
|
||||||
image: managerImage, network: "host", schedule: "*/9 * * * *",
|
|
||||||
args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"],
|
|
||||||
volumes: ["/var/lib/mesh/anthropic-manager:/run/state"],
|
|
||||||
env: {
|
|
||||||
MESH_ANTHROPIC_LICENCE: "personal",
|
|
||||||
MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token",
|
|
||||||
MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage",
|
|
||||||
MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/refresh-token",
|
|
||||||
MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
|
|
||||||
MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token",
|
|
||||||
MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json",
|
|
||||||
MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
});
|
||||||
await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-controller:/anthropic-manager.json`);
|
await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-controller:/anthropic-manager.json`);
|
||||||
await mesh(`module add /anthropic-manager.json`);
|
await mesh(`module add /anthropic-manager.json`);
|
||||||
@@ -329,32 +305,8 @@ test("model access refreshes on the manager node and delivers only the access to
|
|||||||
assert.match(submitted, /sealed to 1 holder/, submitted);
|
assert.match(submitted, /sealed to 1 holder/, submitted);
|
||||||
|
|
||||||
// --- 5. deliver: deploy the consumer and push; it gets the sealed access token -------------------
|
// --- 5. deliver: deploy the consumer and push; it gets the sealed access token -------------------
|
||||||
const consumerManifest = JSON.stringify({
|
// The catalogue's own manifest (novox/hq 04-ISSUES/073).
|
||||||
module: "anthropic-consumer",
|
const consumerManifest = catalogueModule("anthropic-consumer", held);
|
||||||
version: "1",
|
|
||||||
requires: ["model-access"],
|
|
||||||
binds: { "model-access": "/var/lib/anthropic-consumer/model.json" },
|
|
||||||
secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" },
|
|
||||||
"own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" },
|
|
||||||
emits: ["module.anthropic-consumer.usage.session"],
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" },
|
|
||||||
{ id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "apply", type: "container", name: "mesh-anthropic-consumer-apply",
|
|
||||||
image: consumerImage, network: "host", schedule: "*/9 * * * *",
|
|
||||||
args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"],
|
|
||||||
volumes: ["/var/lib/anthropic-consumer:/run/state"],
|
|
||||||
env: {
|
|
||||||
MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token",
|
|
||||||
MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
|
|
||||||
MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json",
|
|
||||||
MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-controller:/anthropic-consumer.json`);
|
await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-controller:/anthropic-consumer.json`);
|
||||||
await mesh(`module add /anthropic-consumer.json`);
|
await mesh(`module add /anthropic-consumer.json`);
|
||||||
await mesh(`module issue anthropic-consumer --node ${MACHINE}`);
|
await mesh(`module issue anthropic-consumer --node ${MACHINE}`);
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -35,7 +35,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "audit-node";
|
const SCENARIO = "audit-node";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -145,26 +145,10 @@ after(async () => {
|
|||||||
test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", {
|
test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", {
|
||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
// The assigned-module manifest (mesh-catalog), its runtime image the ID the machine holds.
|
// The catalogue's manifest (novox/hq 04-ISSUES/073). Its runtime artifact is the image this
|
||||||
const manifest = JSON.stringify({
|
// scenario stocks under the module's slug, `mesh-runtime-audit` — built by scripts/build-runtime-image.sh
|
||||||
module: "audit-logger",
|
// before build-module-runtime.sh generalised it, and named as it was.
|
||||||
version: "1",
|
const manifest = catalogueModule("audit-logger", held, { artifacts: { runtime: "mesh-runtime-audit" } });
|
||||||
consumes: ["#"],
|
|
||||||
"own-secrets": { broker: "/var/lib/audit-logger/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/audit-logger", mode: "0700" },
|
|
||||||
{ id: "trail", type: "directory", path: "/var/lib/audit-logger/trail", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "run", type: "container", name: "mesh-audit-logger", image: pinned("mesh-runtime-audit"),
|
|
||||||
network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/audit-logger/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/audit-logger/trail:/trail",
|
|
||||||
],
|
|
||||||
env: { MESH_BROKER_FILE: "/run/secrets/broker", AUDIT_LOG: "/trail/audit.log" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/audit.json && docker cp /tmp/audit.json mesh-controller:/audit.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/audit.json && docker cp /tmp/audit.json mesh-controller:/audit.json`);
|
||||||
await mesh("module add /audit.json");
|
await mesh("module add /audit.json");
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,221 @@
|
|||||||
|
/**
|
||||||
|
* A machine trusts the mesh's own authority because a module put its root there — and stops when
|
||||||
|
* the module is taken away (novox/hq ADR 0147, 04-ISSUES/129).
|
||||||
|
*
|
||||||
|
* What is dialled is the authority itself. step-ca serves its own API with a leaf it issued, so a
|
||||||
|
* plain client verifying that handshake — no `-k`, no `--cacert` — is verifying exactly one thing:
|
||||||
|
* that this machine's trust store now contains the mesh's root. No proxy, no routed name, no public
|
||||||
|
* issuance. A trust bed leaning on those would pass for their reasons, which is the failure this
|
||||||
|
* whole sequence keeps producing.
|
||||||
|
*
|
||||||
|
* The negative half runs twice, before the module is assigned and after it is unassigned. An anchor
|
||||||
|
* bed that only checks the success would pass on a machine that already trusted everything, and
|
||||||
|
* would say nothing at all about removal — which is the half issue 129 asked for by name.
|
||||||
|
*
|
||||||
|
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||||
|
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||||
|
* MESH_LAB_CATALOG=.../mesh-catalog/modules
|
||||||
|
* step-ca's image is upstream, pinned by the catalogue, pulled by the machine over its uplink.
|
||||||
|
* ca-trust carries no image: a script, a unit, and the machine's own systemd.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { test, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
|
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
|
const capability = await labIsUsable();
|
||||||
|
const binary = hostBinaryPath();
|
||||||
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
|
|
||||||
|
const skip = !capability.usable
|
||||||
|
? `lab not usable: ${capability.why}`
|
||||||
|
: !binary || !existsSync(binary)
|
||||||
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
|
: !bundle || !existsSync(bundle)
|
||||||
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
|
: catalogueIsPresent();
|
||||||
|
|
||||||
|
const SCENARIO = "trust-anchor";
|
||||||
|
const MACHINE = "anchor";
|
||||||
|
/** The authority's own API, which it serves with a certificate it issued itself. */
|
||||||
|
const AUTHORITY = "https://127.0.0.1:9000/health";
|
||||||
|
/** Where the module puts the root, and therefore what removal must take away. */
|
||||||
|
const ANCHOR = "/etc/ca-certificates/trust-source/anchors/mesh-internal-ca.crt";
|
||||||
|
|
||||||
|
let instanceId = "";
|
||||||
|
let held: HeldImage[] = [];
|
||||||
|
|
||||||
|
function quote(s: string): string {
|
||||||
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||||
|
const { stdout } = await exec(instanceId, MACHINE, [
|
||||||
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||||
|
], timeoutMs);
|
||||||
|
const marker = stdout.lastIndexOf("__exit=");
|
||||||
|
if (marker < 0) return { out: stdout, ok: false };
|
||||||
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function must(command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
const { out, ok } = await on(command, timeoutMs);
|
||||||
|
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The control plane, a container on the node. */
|
||||||
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenFrom(said: string): string {
|
||||||
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||||
|
assert.ok(found, `no token in:\n${said}`);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function settled(withinMs = 480_000): Promise<void> {
|
||||||
|
const until = Date.now() + withinMs;
|
||||||
|
let last = "";
|
||||||
|
while (Date.now() < until) {
|
||||||
|
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
||||||
|
if (asked.ok) {
|
||||||
|
try {
|
||||||
|
const state = JSON.parse(asked.out) as {
|
||||||
|
wrong: { node: string; outcome: string }[];
|
||||||
|
waiting: { node: string }[];
|
||||||
|
reported: { node: string; outcome: string; current: boolean }[];
|
||||||
|
};
|
||||||
|
const bad = state.wrong.find((w) => w.node === MACHINE);
|
||||||
|
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
||||||
|
const word = state.reported.find((r) => r.node === MACHINE);
|
||||||
|
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
||||||
|
last = asked.out;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
||||||
|
last = asked.out;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 5000));
|
||||||
|
}
|
||||||
|
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Install a catalogue module's manifest into the control plane, read from the catalogue. */
|
||||||
|
async function register(module: string): Promise<void> {
|
||||||
|
const manifest = catalogueModule(module, held);
|
||||||
|
await must(
|
||||||
|
`printf %s ${quote(manifest)} > /tmp/${module}.json && ` +
|
||||||
|
`docker cp /tmp/${module}.json mesh-controller:/${module}.json`,
|
||||||
|
);
|
||||||
|
await mesh(`module add /${module}.json`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Dial the authority the way anything on this machine would: verifying, with nothing handed to it. */
|
||||||
|
async function verifying(): Promise<{ out: string; ok: boolean }> {
|
||||||
|
return on(`curl --silent --show-error --max-time 10 ${AUTHORITY}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
if (skip) return;
|
||||||
|
|
||||||
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
|
});
|
||||||
|
instanceId = raised.instanceId;
|
||||||
|
held = raised.images;
|
||||||
|
|
||||||
|
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
|
||||||
|
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
||||||
|
const up = await must(`docker ps --format '{{.Names}}'`);
|
||||||
|
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||||
|
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The control plane is a container, and a container that is restarting answers nothing. Asked
|
||||||
|
// until it answers rather than once, so a crash-looping control plane is reported as itself
|
||||||
|
// instead of as whatever command happened to be sent first.
|
||||||
|
let control = { out: "", ok: false };
|
||||||
|
const answering = Date.now() + 120_000;
|
||||||
|
while (Date.now() < answering) {
|
||||||
|
control = await on(`docker exec mesh-controller /mesh-controller status`);
|
||||||
|
if (control.ok) break;
|
||||||
|
await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
}
|
||||||
|
assert.ok(control.ok,
|
||||||
|
`the control plane never answered:\n${control.out}\n` +
|
||||||
|
`${(await on(`docker logs mesh-controller 2>&1 | tail -40`)).out}`);
|
||||||
|
|
||||||
|
await mesh(`node add ${MACHINE}`);
|
||||||
|
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
||||||
|
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
||||||
|
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||||
|
}, { timeout: 1_800_000 });
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
if (instanceId) await destroy(instanceId);
|
||||||
|
await destroyAll(`${SCENARIO}-`);
|
||||||
|
}, { timeout: 600_000 });
|
||||||
|
|
||||||
|
test("the mesh's authority is verified on a machine holding ca-trust, and not on one that is not", {
|
||||||
|
skip, timeout: 1_800_000,
|
||||||
|
}, async () => {
|
||||||
|
// The authority first, on its own. Nothing about trust yet.
|
||||||
|
await register("step-ca");
|
||||||
|
await mesh(`module issue step-ca --node ${MACHINE}`);
|
||||||
|
await mesh(`assign ${MACHINE} step-ca`);
|
||||||
|
await mesh(`push ${MACHINE}`);
|
||||||
|
await settled();
|
||||||
|
|
||||||
|
// It is up and answering — asked the way nothing else in this bed is allowed to ask, with
|
||||||
|
// verification off, because at this point in the bed no machine could verify it.
|
||||||
|
let answering = false;
|
||||||
|
const until = Date.now() + 180_000;
|
||||||
|
while (Date.now() < until && !answering) {
|
||||||
|
answering = (await on(`curl --silent --insecure --max-time 5 ${AUTHORITY}`)).ok;
|
||||||
|
if (!answering) await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
}
|
||||||
|
assert.ok(answering, `the authority never answered:\n${(await on(`docker logs step-ca 2>&1 | tail -30`)).out}`);
|
||||||
|
|
||||||
|
// **The negative half, before anything is assigned.** If this passes, the bed is measuring
|
||||||
|
// something already in the machine's trust store and everything below it is worthless.
|
||||||
|
const before = await verifying();
|
||||||
|
assert.equal(before.ok, false, `the authority verified before anything anchored its root:\n${before.out}`);
|
||||||
|
assert.match(before.out, /certificate|issuer/i, `it failed for some other reason:\n${before.out}`);
|
||||||
|
|
||||||
|
// Now the module.
|
||||||
|
await register("ca-trust");
|
||||||
|
await mesh(`assign ${MACHINE} ca-trust`);
|
||||||
|
await mesh(`push ${MACHINE}`);
|
||||||
|
await settled();
|
||||||
|
|
||||||
|
const unit = await on(`systemctl is-active mesh-ca-trust.service`);
|
||||||
|
assert.ok(unit.ok, `the unit is ${unit.out.trim()}:\n${(await on(`journalctl -u mesh-ca-trust --no-pager | tail -30`)).out}`);
|
||||||
|
await must(`test -s ${ANCHOR}`);
|
||||||
|
const anchors = await must(`trust list | grep -A2 -i 'Mesh Internal CA' || trust list`);
|
||||||
|
assert.match(anchors, /Mesh Internal CA/, `the mesh's authority is not among the machine's anchors:\n${anchors}`);
|
||||||
|
|
||||||
|
// **The whole claim, in one command.** No -k, no --cacert: the machine's own trust store, and a
|
||||||
|
// certificate the mesh's authority issued.
|
||||||
|
const after = await verifying();
|
||||||
|
assert.ok(after.ok, `the authority still does not verify with the module assigned:\n${after.out}`);
|
||||||
|
|
||||||
|
// **And removal is the same unit's business.** Unassigned, the host stops it; stopping it takes
|
||||||
|
// the anchor away and refreshes the bundles, so the machine stops trusting the mesh.
|
||||||
|
await mesh(`unassign ${MACHINE} ca-trust`);
|
||||||
|
await mesh(`push ${MACHINE}`);
|
||||||
|
await settled();
|
||||||
|
|
||||||
|
const gone = await on(`test -e ${ANCHOR}`);
|
||||||
|
assert.equal(gone.ok, false, "the anchor is still on the machine after the module was unassigned");
|
||||||
|
const afterwards = await verifying();
|
||||||
|
assert.equal(afterwards.ok, false, `the machine still verifies the mesh's authority with nothing anchoring it:\n${afterwards.out}`);
|
||||||
|
assert.match(afterwards.out, /certificate|issuer/i, `it failed for some other reason:\n${afterwards.out}`);
|
||||||
|
});
|
||||||
@@ -36,7 +36,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -49,7 +49,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "catalogue-mqtt";
|
const SCENARIO = "catalogue-mqtt";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -158,101 +158,11 @@ test("the mesh assigns mosquitto: a run-once step seeds dynsec before the broker
|
|||||||
skip, timeout: 1_500_000,
|
skip, timeout: 1_500_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
// mosquitto's dynsec config: the plugin refuses to start unless dynamic-security.json holds an
|
// mosquitto's dynsec config: the plugin refuses to start unless dynamic-security.json holds an
|
||||||
// admin client, so the store MUST be seeded first. The `run-once` bootstrap container is declared
|
// admin client, so the store MUST be seeded first. The catalogue's manifest declares a `run-once`
|
||||||
// BEFORE `server` (the broker) and reuses the module's runtime image; the host runs it to
|
// bootstrap container BEFORE `server` (the broker), reusing the module's runtime image; the host
|
||||||
// completion, then starts the broker. The runtime `server`/`runtime` shape mirrors the committed
|
// runs it to completion, then starts the broker. The manifest is the catalogue's own, its runtime
|
||||||
// manifest, with images pinned to what this scenario serves by digest.
|
// artifact the image this scenario stocked (novox/hq 04-ISSUES/073).
|
||||||
const mosquittoConf =
|
const manifest = catalogueModule("mosquitto", held);
|
||||||
"persistence true\n" +
|
|
||||||
"persistence_location /mosquitto/data\n\n" +
|
|
||||||
"log_dest stdout\n" +
|
|
||||||
"log_type warning\n" +
|
|
||||||
"log_type error\n" +
|
|
||||||
"log_type notice\n\n" +
|
|
||||||
"# Every client authenticates; identities and their per-topic ACLs are managed\n" +
|
|
||||||
"# at runtime by the dynamic security plugin, whose store the plugin itself owns.\n" +
|
|
||||||
"allow_anonymous false\n" +
|
|
||||||
"plugin /usr/lib/mosquitto_dynamic_security.so\n" +
|
|
||||||
"plugin_opt_config_file /mosquitto/data/dynamic-security.json\n\n" +
|
|
||||||
"# MQTT listener\n" +
|
|
||||||
"listener 1883\n\n" +
|
|
||||||
"# MQTT-over-WebSockets listener\n" +
|
|
||||||
"listener 8081\n" +
|
|
||||||
"protocol websockets\n";
|
|
||||||
|
|
||||||
const manifest = JSON.stringify({
|
|
||||||
module: "mosquitto",
|
|
||||||
version: "1",
|
|
||||||
provides: [{ name: "mqtt-topic", scope: "mesh" }],
|
|
||||||
serves: { "mqtt-topic": {} },
|
|
||||||
emits: ["module.mosquitto.topic.provisioned", "module.mosquitto.topic.deprovisioned"],
|
|
||||||
// The events entrypoint subscribes to its own lifecycle events (an audit log), so it consumes
|
|
||||||
// them too — declared, or the foundation never makes the queue the runtime binds (ADR 0046).
|
|
||||||
consumes: ["module.mosquitto.topic.provisioned", "module.mosquitto.topic.deprovisioned"],
|
|
||||||
receives: { "mqtt-topic": "/var/lib/mosquitto-module/grants/mesh.json" },
|
|
||||||
grants: { "mqtt-topic": "/var/lib/mosquitto-module/grants" },
|
|
||||||
"own-secrets": {
|
|
||||||
admin: "/var/lib/mosquitto-module/admin.secret",
|
|
||||||
broker: "/var/lib/mesh/mosquitto/broker",
|
|
||||||
},
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/mosquitto", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/mosquitto-module", mode: "0700" },
|
|
||||||
{ id: "grants-dir", type: "directory", path: "/var/lib/mosquitto-module/grants", mode: "0700" },
|
|
||||||
// The broker runs as uid 1883, so the shared data directory it seeds into and persists to is
|
|
||||||
// its own.
|
|
||||||
{ id: "data", type: "directory", path: "/services/mosquitto/data", mode: "0700", owner: "1883:1883" },
|
|
||||||
{
|
|
||||||
id: "server-conf", type: "file", path: "/var/lib/mosquitto-module/mosquitto.conf",
|
|
||||||
mode: "0600", owner: "1883:1883", content: mosquittoConf,
|
|
||||||
},
|
|
||||||
{ id: "net", type: "network", name: "mosquitto" },
|
|
||||||
// THE run-once step: seed dynsec offline, once, before the broker. It reuses the runtime image
|
|
||||||
// (`mesh-tools run <bootstrap>` imports mosquitto's bootstrap entrypoint, which writes the
|
|
||||||
// admin client into dynamic-security.json and chowns it to the broker's uid, then exits). It is
|
|
||||||
// declared BEFORE `server`; the host runs it to completion and requires exit 0 before starting
|
|
||||||
// the broker.
|
|
||||||
{
|
|
||||||
id: "bootstrap", type: "container", name: "mosquitto-bootstrap",
|
|
||||||
image: pinned("mesh-runtime-mosquitto"), "run-once": true,
|
|
||||||
volumes: [
|
|
||||||
"/services/mosquitto/data:/mosquitto/data",
|
|
||||||
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_PROVISION_MQTT: "mosquitto:1883",
|
|
||||||
MESH_PROVISION_ADMIN_USER: "mesh-admin",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/admin",
|
|
||||||
MESH_DYNSEC_FILE: "/mosquitto/data/dynamic-security.json",
|
|
||||||
},
|
|
||||||
args: ["run", "/app/modules/mosquitto/dist/bootstrap/index.js"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "mosquitto", image: pinned("eclipse-mosquitto"),
|
|
||||||
network: "mosquitto", ports: ["1883", "8081"],
|
|
||||||
volumes: [
|
|
||||||
"/services/mosquitto/data:/mosquitto/data",
|
|
||||||
"/var/lib/mosquitto-module/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro",
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-mosquitto",
|
|
||||||
image: pinned("mesh-runtime-mosquitto"), network: "mosquitto",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/mosquitto/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/mosquitto-module/grants:/var/lib/mosquitto-module/grants:ro",
|
|
||||||
"/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/mosquitto-module/grants/mesh.json",
|
|
||||||
MESH_PROVISION_MQTT: "mosquitto:1883",
|
|
||||||
MESH_PROVISION_ADMIN_USER: "mesh-admin",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/admin",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/mosquitto.json && docker cp /tmp/mosquitto.json mesh-controller:/mosquitto.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/mosquitto.json && docker cp /tmp/mosquitto.json mesh-controller:/mosquitto.json`);
|
||||||
await mesh("module add /mosquitto.json");
|
await mesh("module add /mosquitto.json");
|
||||||
|
|||||||
@@ -1,210 +0,0 @@
|
|||||||
/**
|
|
||||||
* The mesh assigns grafana's tool runtime, configured entirely by the assignment's settings — the
|
|
||||||
* ADR 0051 + 0052 case: config is the assignment's, delivered as a settings-merged file the runtime
|
|
||||||
* reads, not a credential baked into the manifest.
|
|
||||||
*
|
|
||||||
* plex/sonarr prove a runtime that self-detects its key from the app's own config. This proves the
|
|
||||||
* other half: the operator states grafana's URL and an API token as settings for this node, the
|
|
||||||
* control plane merges them into the module's mergeable config file, and the runtime reads that file
|
|
||||||
* at start, registers grafana's tools, and serves them under its scoped account. There is no live
|
|
||||||
* Grafana — that the serve queue is bound is the proof the settings reached the runtime and its
|
|
||||||
* tools loaded from them.
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh grafana builds mesh-runtime-grafana:development into the local
|
|
||||||
* daemon, which scenarios/grafana-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "grafana-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("the mesh assigns grafana's runtime, configured by settings, and it serves its tools", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// A grafana manifest with no credential in it: its runtime, and a mergeable config file the
|
|
||||||
// settings will fill. This is the whole point of ADR 0051 — the manifest carries defaults and
|
|
||||||
// structure, the assignment carries the URL and token.
|
|
||||||
const manifest = JSON.stringify({
|
|
||||||
module: "grafana",
|
|
||||||
version: "1",
|
|
||||||
emits: ["module.grafana.alert.firing"],
|
|
||||||
"own-secrets": { broker: "/var/lib/mesh/grafana/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/grafana", mode: "0700" },
|
|
||||||
{ id: "config", type: "file", path: "/var/lib/mesh/grafana/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-grafana", image: pinned("mesh-runtime-grafana"),
|
|
||||||
network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/grafana/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/mesh/grafana/config.json:/run/config/config.json:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_GRAFANA_CONFIG_FILE: "/run/config/config.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/grafana.json && docker cp /tmp/grafana.json mesh-controller:/grafana.json`);
|
|
||||||
await mesh("module add /grafana.json");
|
|
||||||
|
|
||||||
// The operator states grafana's URL and API token as settings for this node — the config the
|
|
||||||
// runtime will read. Nothing about them is in the manifest.
|
|
||||||
const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token: "lab-grafana-token" });
|
|
||||||
await must(`printf %s ${quote(settings)} > /tmp/grafana-settings.json && docker cp /tmp/grafana-settings.json mesh-controller:/grafana-settings.json`);
|
|
||||||
await mesh(`settings set grafana /grafana-settings.json --node ${MACHINE}`);
|
|
||||||
|
|
||||||
const issued = await mesh(`module issue grafana --node ${MACHINE}`);
|
|
||||||
assert.match(issued, /scoped to what it emits and consumes/, issued);
|
|
||||||
await mesh(`assign ${MACHINE} grafana`);
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
const running = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
assert.match(running, /mesh-grafana/,
|
|
||||||
`grafana's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
|
|
||||||
|
|
||||||
// The settings reached the node: the rendered config file carries what was set, not the manifest's
|
|
||||||
// empty default.
|
|
||||||
const config = await must(`cat /var/lib/mesh/grafana/config.json`);
|
|
||||||
assert.match(config, /lab-grafana-token/, `the settings did not merge into the config file:\n${config}`);
|
|
||||||
|
|
||||||
const credential = await must(`cat /var/lib/mesh/grafana/broker`);
|
|
||||||
assert.match(credential, /"url":"amqps:\/\/anchor-grafana:/, `not the scoped account:\n${credential}`);
|
|
||||||
assert.doesNotMatch(credential, /guest:guest/, "grafana's runtime holds the broker's own account");
|
|
||||||
|
|
||||||
// The runtime read that config, built its client from the settings-provided token, registered its
|
|
||||||
// tools, and bound their serve queues — the queue on the broker is the proof the settings-config
|
|
||||||
// path reached serving, with no credential in the manifest and no live Grafana.
|
|
||||||
let served = "";
|
|
||||||
const untilServing = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilServing) {
|
|
||||||
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
|
|
||||||
if (/serve\.grafana\.grafana_status/.test(served)) break;
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(served, /serve\.grafana\.grafana_status/,
|
|
||||||
`grafana's runtime never bound its serve queue (settings not read?):\n` +
|
|
||||||
`${(await on(`docker logs mesh-grafana 2>&1 | tail -20`)).out}\n---\n${served}`);
|
|
||||||
|
|
||||||
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
|
||||||
assert.match(users, /anchor-grafana/, `the scoped account is not on the broker:\n${users}`);
|
|
||||||
});
|
|
||||||
@@ -38,7 +38,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -51,7 +51,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "model-usage-bed";
|
const SCENARIO = "model-usage-bed";
|
||||||
/** The node that carries the postgres provider and the model-usage consumer. anchor carries only the
|
/** The node that carries the postgres provider and the model-usage consumer. anchor carries only the
|
||||||
@@ -190,7 +190,9 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot
|
|||||||
}, async () => {
|
}, async () => {
|
||||||
// ================================================================================================
|
// ================================================================================================
|
||||||
// THE MANIFESTS — postgres verbatim from two-node-db (its server publishes 5432 so the consumer
|
// THE MANIFESTS — postgres verbatim from two-node-db (its server publishes 5432 so the consumer
|
||||||
// reaches it), and model-usage the committed catalogue shape with its images pinned.
|
// reaches it): a SECOND postgres beside the foundation's store, which the catalogue's postgres would
|
||||||
|
// instead claim and adopt in place. Still an inline copy, declared in beds-read-the-catalogue.test.ts
|
||||||
|
// (novox/hq 04-ISSUES/073). model-usage is the catalogue's.
|
||||||
// ================================================================================================
|
// ================================================================================================
|
||||||
const postgresManifest = JSON.stringify({
|
const postgresManifest = JSON.stringify({
|
||||||
module: "postgres",
|
module: "postgres",
|
||||||
@@ -233,45 +235,12 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot
|
|||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
// --- model-usage: requires postgres-database, owns a provisioned store, consumes module.*.usage.*,
|
// --- model-usage: the catalogue's own manifest (novox/hq 04-ISSUES/073). It requires
|
||||||
// runs a run-once migrate then the long-lived event consumer. Both containers on the host network so
|
// postgres-database, owns a provisioned store, consumes module.*.usage.*, and its runtime is on the
|
||||||
// they reach the granted postgres (at the provider's address the mesh writes) and the broker. ------
|
// host network so it reaches the granted postgres (at the provider's address the mesh writes) and
|
||||||
const modelUsageManifest = JSON.stringify({
|
// the broker. Its slug keeps the consumer identity under the 20 characters an S3 access key allows
|
||||||
module: "model-usage",
|
// (ADR 0049). ------------------------------------------------------------------------------------
|
||||||
version: "1",
|
const modelUsageManifest = catalogueModule("model-usage", held);
|
||||||
// `mesh_laptop_model-usage` is 23 chars, over the 20 an S3 access key keeps (ADR 0049); a short
|
|
||||||
// slug makes the consumer identity `mesh_laptop_usage` (17). db/role/`as` all derive from it.
|
|
||||||
slug: "usage",
|
|
||||||
capabilities: ["container-runtime"],
|
|
||||||
requires: ["postgres-database"],
|
|
||||||
contributes: { "postgres-database": { name: "model_usage" } },
|
|
||||||
binds: { "postgres-database": "/var/lib/model-usage/database.json" },
|
|
||||||
secrets: { "postgres-database": "/var/lib/model-usage/database.secret" },
|
|
||||||
consumes: ["module.*.usage.*"],
|
|
||||||
"own-secrets": { broker: "/var/lib/mesh/model-usage/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/model-usage", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/model-usage", mode: "0700" },
|
|
||||||
// The connection string carries the password, so it reaches the runtime as a file the mesh
|
|
||||||
// templates (novox/hq ADR 0086), the shape the catalogue's manifest has.
|
|
||||||
{
|
|
||||||
id: "database-url", type: "file", path: "/var/lib/model-usage/database.url", mode: "0600",
|
|
||||||
content:
|
|
||||||
"postgresql://${bound:postgres-database:as}:${secret:postgres-database}@" +
|
|
||||||
"${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-model-usage",
|
|
||||||
image: pinned("mesh-runtime-model-usage"), network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/model-usage/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/model-usage:/run/state",
|
|
||||||
"/var/lib/model-usage/database.url:/run/secrets/database-url:ro",
|
|
||||||
],
|
|
||||||
env: { MESH_BROKER_FILE: "/run/secrets/broker", DATABASE_URL_FILE: "/run/secrets/database-url" },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
async function addIssueAssign(name: string, manifest: string): Promise<void> {
|
async function addIssueAssign(name: string, manifest: string): Promise<void> {
|
||||||
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
||||||
|
|||||||
@@ -1,231 +0,0 @@
|
|||||||
/**
|
|
||||||
* The mesh assigns plex's tool runtime, and it serves plex's tools over an account the mesh
|
|
||||||
* delivered — the whole of novox/hq ADR 0052.
|
|
||||||
*
|
|
||||||
* assigned-audit proves an assigned *consumer* (ADR 0048). This proves an assigned module that runs
|
|
||||||
* its OWN code as its OWN process under its OWN scoped account and *serves tools*: the module is
|
|
||||||
* assigned through the control plane, the mesh issues it an account scoped to serve.plex.* (and its
|
|
||||||
* events), seals it to the machine, and the host runs it as a container that binds amqps with that
|
|
||||||
* account. A caller then invokes plex.plex_reachable over the mesh and gets the tool's own answer —
|
|
||||||
* proof the invocation routed to the assigned runtime, ran plex's real code, and replied, all under
|
|
||||||
* the scoped account and never the broker's own.
|
|
||||||
*
|
|
||||||
* It needs the host binary, the foundation bundle, and the runtime image the scenario loads:
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
|
||||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh plex builds mesh-runtime-plex:development into the local daemon,
|
|
||||||
* which scenarios/plex-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "plex-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
/** The mesh's own images, as the machines hold them. */
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The control plane, a container on the node. */
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The reference a manifest should carry, once this scenario has been raised. */
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
// Raise the foundation — store, broker, control — from the bundle.
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// The node joins its own mesh, so it is a node the mesh can assign to, and start the host so it
|
|
||||||
// applies what it is pushed.
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("the mesh assigns plex's runtime, and it serves plex's tools over the account the mesh delivered", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// A minimal plex manifest: its tools/events runtime (no Plex server or media mounts in the lab),
|
|
||||||
// its emits and consumes so the account is scoped to those too, and a token in the environment so
|
|
||||||
// the tools register without a running Plex to detect one from. The runtime image is the digest
|
|
||||||
// this scenario's registry serves.
|
|
||||||
const manifest = JSON.stringify({
|
|
||||||
module: "plex",
|
|
||||||
version: "1",
|
|
||||||
emits: [
|
|
||||||
"module.plex.playback.started",
|
|
||||||
"module.plex.playback.stopped",
|
|
||||||
"module.plex.item.added",
|
|
||||||
],
|
|
||||||
consumes: ["module.*.download.completed"],
|
|
||||||
"own-secrets": { broker: "/var/lib/mesh/plex/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/plex", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-plex", image: pinned("mesh-runtime-plex"),
|
|
||||||
network: "host",
|
|
||||||
volumes: ["/var/lib/mesh/plex/broker:/run/secrets/broker:ro"],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_PLEX_URL: "http://127.0.0.1:32400",
|
|
||||||
MESH_PLEX_TOKEN: "lab-token",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/plex.json && docker cp /tmp/plex.json mesh-controller:/plex.json`);
|
|
||||||
await mesh("module add /plex.json");
|
|
||||||
|
|
||||||
// The mesh issues plex's scoped account and seals it to this machine, then assigns and pushes it.
|
|
||||||
const issued = await mesh(`module issue plex --node ${MACHINE}`);
|
|
||||||
assert.match(issued, /scoped to what it emits and consumes/, issued);
|
|
||||||
await mesh(`assign ${MACHINE} plex`);
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
// The runtime container the mesh started is running.
|
|
||||||
const running = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
assert.match(running, /mesh-plex/,
|
|
||||||
`plex's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
|
|
||||||
|
|
||||||
// The credential on disk is the scoped account over amqps, sealed — not the broker's own.
|
|
||||||
const credential = await must(`cat /var/lib/mesh/plex/broker`);
|
|
||||||
assert.match(credential, /"url":"amqps:\/\/anchor-plex:/, `not the scoped account:\n${credential}`);
|
|
||||||
assert.doesNotMatch(credential, /guest:guest/, "plex's runtime holds the broker's own account");
|
|
||||||
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker");
|
|
||||||
|
|
||||||
// The runtime registered and is serving its tools — the queue it declared is on the broker,
|
|
||||||
// named for the scope its account is granted (serve.plex.*).
|
|
||||||
let served = "";
|
|
||||||
const untilServing = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilServing) {
|
|
||||||
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
|
|
||||||
if (/serve\.plex\.plex_reachable/.test(served)) break;
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(served, /serve\.plex\.plex_reachable/,
|
|
||||||
`plex's runtime never bound its serve queue:\n${(await on(`docker logs mesh-plex 2>&1 | tail -20`)).out}\n---\n${served}`);
|
|
||||||
|
|
||||||
// A caller invokes plex.plex_reachable over the mesh, from the bootstrap account (a caller, like
|
|
||||||
// mesh-controller's command API — plex's own account serves, it does not call). The reply is the
|
|
||||||
// tool's own answer: it ran in the assigned runtime and reported the Plex server is unreachable
|
|
||||||
// (there is none in the lab). A reply at all — not a timeout — is the proof the invocation routed
|
|
||||||
// to the assigned runtime and ran plex's real code under its scoped account.
|
|
||||||
const invoked = await must(
|
|
||||||
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
|
|
||||||
`${pinned("mesh-runtime-plex")} invoke plex plex_reachable`,
|
|
||||||
120_000,
|
|
||||||
);
|
|
||||||
const line = invoked.split("\n").map((l) => l.trim()).filter(Boolean).pop() ?? "";
|
|
||||||
const result = JSON.parse(line) as { reachable: boolean; url: string; error?: string };
|
|
||||||
assert.equal(result.reachable, false, `expected the lab's Plex to be unreachable:\n${invoked}`);
|
|
||||||
assert.match(result.url, /127\.0\.0\.1:32400/, `the tool ran but not against the configured server:\n${invoked}`);
|
|
||||||
|
|
||||||
// And the account the mesh made for it is a real one on the broker, scoped — proven above by the
|
|
||||||
// serve queue authenticating and the invocation round-tripping under it.
|
|
||||||
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
|
||||||
assert.match(users, /anchor-plex/, `the scoped account is not on the broker:\n${users}`);
|
|
||||||
});
|
|
||||||
@@ -1,273 +0,0 @@
|
|||||||
/**
|
|
||||||
* The mesh assigns redis — a *provider* — and its runtime runs the provisioner AND the tools as one
|
|
||||||
* process under one account the mesh delivered (novox/hq ADR 0052).
|
|
||||||
*
|
|
||||||
* assigned-plex proves an assigned module that serves tools. This proves the provider half: redis's
|
|
||||||
* runtime binds its scoped account, serves redis's tools against the real server (redis_ping →
|
|
||||||
* PONG), and — the thing 0052 fixes — runs its provisioner in that same broker-bound process, so a
|
|
||||||
* grant is provisioned and its lifecycle event is emitted onto the mesh. Before 0052 the provisioner
|
|
||||||
* ran in a container with no broker and its emit could not fire at all.
|
|
||||||
*
|
|
||||||
* A caveat this test makes explicit: runProvisioner needs a seal key ($MESH_SEAL_KEY) and the mesh
|
|
||||||
* has no way yet to deliver one to a provider's runtime (04-ISSUES). The manifest here sets a
|
|
||||||
* lab-local key so the mechanism can be proven; the delivery is a separate, open design question.
|
|
||||||
*
|
|
||||||
* It needs the host binary, the foundation bundle, and the runtime image the scenario loads:
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development into the local
|
|
||||||
* daemon, which scenarios/redis-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "redis-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("the mesh assigns redis, and its runtime serves tools and provisions grants over the account the mesh delivered", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// A redis manifest with both halves it needs on this node: the redis server, and one broker-bound
|
|
||||||
// runtime that serves redis's tools AND runs its provisioner. Both reach the server over the host
|
|
||||||
// (127.0.0.1:6379) with the same admin password the mesh generated. MESH_SEAL_KEY is lab-local —
|
|
||||||
// the mesh cannot yet deliver one to a provider's runtime (see the file header / 04-ISSUES).
|
|
||||||
const manifest = JSON.stringify({
|
|
||||||
module: "redis",
|
|
||||||
version: "1",
|
|
||||||
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
// redis's events entrypoint subscribes to its own lifecycle events (an audit-trail log), so it
|
|
||||||
// consumes them too — declared, or the foundation never makes the queue the runtime binds and it
|
|
||||||
// crashes on start with a 404 (novox/hq ADR 0046: a consume is declared).
|
|
||||||
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
|
|
||||||
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
|
|
||||||
{
|
|
||||||
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
|
|
||||||
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/services/redis/data:/data",
|
|
||||||
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
|
|
||||||
],
|
|
||||||
args: ["/etc/redis/redis.conf"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
|
|
||||||
network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
|
|
||||||
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
GRANTS: "/var/lib/redis-module/grants",
|
|
||||||
MESH_PROVISION_REDIS: "127.0.0.1:6379",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
|
|
||||||
MESH_SEAL_KEY: "lab-only-seal-key",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
|
|
||||||
await mesh("module add /redis.json");
|
|
||||||
|
|
||||||
const issued = await mesh(`module issue redis --node ${MACHINE}`);
|
|
||||||
assert.match(issued, /scoped to what it emits and consumes/, issued);
|
|
||||||
await mesh(`assign ${MACHINE} redis`);
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
// The server and the runtime the mesh started are both running.
|
|
||||||
const running = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
assert.match(running, /\bredis\b/, `redis's server is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
|
|
||||||
assert.match(running, /mesh-redis/, `redis's runtime is not running:\n${(await on(`docker logs mesh-redis 2>&1 | tail -20`)).out}`);
|
|
||||||
|
|
||||||
// The credential on disk is the scoped account over amqps, sealed — not the broker's own.
|
|
||||||
const credential = await must(`cat /var/lib/mesh/redis/broker`);
|
|
||||||
assert.match(credential, /"url":"amqps:\/\/anchor-redis:/, `not the scoped account:\n${credential}`);
|
|
||||||
assert.doesNotMatch(credential, /guest:guest/, "redis's runtime holds the broker's own account");
|
|
||||||
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker");
|
|
||||||
|
|
||||||
// The runtime registered and is serving its tools — the serve queue is on the broker.
|
|
||||||
let served = "";
|
|
||||||
const untilServing = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilServing) {
|
|
||||||
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
|
|
||||||
if (/serve\.redis\.redis_ping/.test(served)) break;
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(served, /serve\.redis\.redis_ping/,
|
|
||||||
`redis's runtime never bound its serve queue:\n${(await on(`docker logs mesh-redis 2>&1 | tail -30`)).out}\n---\n${served}`);
|
|
||||||
|
|
||||||
// A caller invokes redis.redis_ping over the mesh: the tool runs in the assigned runtime, reaches
|
|
||||||
// the real redis, and answers PONG. A positive round-trip against a real backend.
|
|
||||||
const pinged = await must(
|
|
||||||
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
|
|
||||||
`${pinned("mesh-runtime-redis")} invoke redis redis_ping`,
|
|
||||||
120_000,
|
|
||||||
);
|
|
||||||
const pingLine = pinged.split("\n").map((l) => l.trim()).filter(Boolean).pop() ?? "";
|
|
||||||
const ping = JSON.parse(pingLine) as { ok: unknown };
|
|
||||||
assert.ok(String(ping.ok).toUpperCase().includes("PONG") || ping.ok === true,
|
|
||||||
`redis_ping did not answer PONG through the mesh:\n${pinged}`);
|
|
||||||
|
|
||||||
// The provider path: a grant appears (as the control plane would write it), and the provisioner —
|
|
||||||
// running inside the same broker-bound runtime — creates the ACL user and emits the lifecycle
|
|
||||||
// event. The sealed credential the harness writes only after adapter.create() returns is the
|
|
||||||
// proof create() ran to completion; and because emit() awaits the broker's publish confirm
|
|
||||||
// (ADR 0047), a completed create() means the provisioned event was accepted onto the mesh.
|
|
||||||
const grant = JSON.stringify({ resource: "redis-cache", consumer: "app-one", node: MACHINE, values: {} });
|
|
||||||
await must(`printf %s ${quote(grant)} > /var/lib/redis-module/grants/app-one.grant.json`);
|
|
||||||
|
|
||||||
let credentialWritten = false;
|
|
||||||
const untilProvisioned = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilProvisioned) {
|
|
||||||
const ls = await on(`ls /var/lib/redis-module/grants/`);
|
|
||||||
if (ls.ok && /app-one\.redis-cache\.credential/.test(ls.out)) { credentialWritten = true; break; }
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.ok(credentialWritten,
|
|
||||||
`the provisioner never provisioned the grant (no emit under a bound broker?):\n` +
|
|
||||||
`${(await on(`docker logs mesh-redis 2>&1 | tail -30`)).out}`);
|
|
||||||
|
|
||||||
// No emit failed: the provisioner's announce() logs "emit ... failed" only when the broker refused
|
|
||||||
// the publish. Its absence, with the credential written, is the provisioner emitting on the mesh.
|
|
||||||
const runtimeLog = (await on(`docker logs mesh-redis 2>&1`)).out;
|
|
||||||
assert.doesNotMatch(runtimeLog, /emit .*failed/,
|
|
||||||
`the provisioner's emit was refused — the account cannot publish its lifecycle event:\n${runtimeLog}`);
|
|
||||||
|
|
||||||
// And the ACL user the provisioner created is really on the redis server — the provisioning did
|
|
||||||
// its own half, not only the mesh bookkeeping. Asked through the same served tool surface.
|
|
||||||
const acl = await must(
|
|
||||||
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
|
|
||||||
`${pinned("mesh-runtime-redis")} invoke redis redis_command '{"command":"ACL LIST"}'`,
|
|
||||||
120_000,
|
|
||||||
);
|
|
||||||
assert.match(acl, /app-one/, `the provisioner did not create the consumer's ACL user on redis:\n${acl}`);
|
|
||||||
|
|
||||||
// The scoped account the mesh made for it is a real one on the broker.
|
|
||||||
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
|
||||||
assert.match(users, /anchor-redis/, `the scoped account is not on the broker:\n${users}`);
|
|
||||||
});
|
|
||||||
@@ -1,214 +0,0 @@
|
|||||||
/**
|
|
||||||
* The mesh assigns sonarr's tool runtime, and it serves sonarr's tools over an account the mesh
|
|
||||||
* delivered — the Servarr case of novox/hq ADR 0052.
|
|
||||||
*
|
|
||||||
* assigned-plex proved a tools+events module that self-detects its token from a mounted config dir.
|
|
||||||
* This proves that self-configuring pattern generalises to the Servarr family: sonarr's runtime
|
|
||||||
* detects its API key from the server's own config.xml (a file resource stands in for the running
|
|
||||||
* Sonarr here), registers its tools, and serves them under a scoped account. There is no live Sonarr
|
|
||||||
* to reach — that the serve queue is bound is the proof the key was detected and the tools loaded.
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh sonarr builds mesh-runtime-sonarr:development into the local
|
|
||||||
* daemon, which scenarios/sonarr-node.yml stocks — so no MESH_LAB_RUNTIME here; the host pulls it.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "sonarr-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("the mesh assigns sonarr's runtime, and it detects its key and serves its tools", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// A minimal sonarr manifest: its tool runtime, and a config.xml the runtime detects its API key
|
|
||||||
// from — the file resource stands in for the running Sonarr that would write it. No Sonarr server
|
|
||||||
// or media mounts; the tools simply have nothing live to reach.
|
|
||||||
const manifest = JSON.stringify({
|
|
||||||
module: "sonarr",
|
|
||||||
version: "1",
|
|
||||||
emits: ["module.sonarr.episode.grabbed", "module.sonarr.download.completed"],
|
|
||||||
consumes: [],
|
|
||||||
"own-secrets": { broker: "/var/lib/mesh/sonarr/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/sonarr", mode: "0700" },
|
|
||||||
{ id: "config", type: "directory", path: "/services/sonarr/config", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "config-xml", type: "file", path: "/services/sonarr/config/config.xml", mode: "0644",
|
|
||||||
content: "<Config>\n <Port>8989</Port>\n <ApiKey>labdetectedapikey0000000000000000</ApiKey>\n</Config>\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-sonarr", image: pinned("mesh-runtime-sonarr"),
|
|
||||||
network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/sonarr/broker:/run/secrets/broker:ro",
|
|
||||||
"/services/sonarr/config:/var/lib/sonarr/config:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_SONARR_URL: "http://127.0.0.1:8989",
|
|
||||||
MESH_SONARR_CONFIG_DIR: "/var/lib/sonarr/config",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/sonarr.json && docker cp /tmp/sonarr.json mesh-controller:/sonarr.json`);
|
|
||||||
await mesh("module add /sonarr.json");
|
|
||||||
|
|
||||||
const issued = await mesh(`module issue sonarr --node ${MACHINE}`);
|
|
||||||
assert.match(issued, /scoped to what it emits and consumes/, issued);
|
|
||||||
await mesh(`assign ${MACHINE} sonarr`);
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
const running = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
assert.match(running, /mesh-sonarr/,
|
|
||||||
`sonarr's runtime was assigned and is not running:\n${(await on(`tail -30 /var/log/mesh-host.log`)).out}`);
|
|
||||||
|
|
||||||
const credential = await must(`cat /var/lib/mesh/sonarr/broker`);
|
|
||||||
assert.match(credential, /"url":"amqps:\/\/anchor-sonarr:/, `not the scoped account:\n${credential}`);
|
|
||||||
assert.doesNotMatch(credential, /guest:guest/, "sonarr's runtime holds the broker's own account");
|
|
||||||
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/, "no fingerprint to pin the broker");
|
|
||||||
|
|
||||||
// The runtime detected its API key from config.xml, registered its tools, and bound their serve
|
|
||||||
// queues — the queue on the broker is the proof the whole chain worked with no live Sonarr.
|
|
||||||
let served = "";
|
|
||||||
const untilServing = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilServing) {
|
|
||||||
served = await must(`docker exec mesh-broker lavinmqctl list_queues name 2>&1 || true`);
|
|
||||||
if (/serve\.sonarr\.sonarr_status/.test(served)) break;
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(served, /serve\.sonarr\.sonarr_status/,
|
|
||||||
`sonarr's runtime never bound its serve queue (key not detected?):\n` +
|
|
||||||
`${(await on(`docker logs mesh-sonarr 2>&1 | tail -20`)).out}\n---\n${served}`);
|
|
||||||
|
|
||||||
// A caller invokes sonarr_status over the mesh: it routes to the assigned runtime, which runs
|
|
||||||
// sonarr's real code and reports Sonarr unreachable (there is none). A reply — not a timeout — is
|
|
||||||
// the proof the invocation reached the runtime under its scoped account.
|
|
||||||
const invoked = await on(
|
|
||||||
`docker run --rm --network host -e MESH_BROKER_URL=amqp://guest:guest@127.0.0.1:5672/ ` +
|
|
||||||
`${pinned("mesh-runtime-sonarr")} invoke sonarr sonarr_status`,
|
|
||||||
120_000,
|
|
||||||
);
|
|
||||||
assert.doesNotMatch(invoked.out, /timed out/,
|
|
||||||
`sonarr_status timed out — nothing served the invocation:\n${invoked.out}`);
|
|
||||||
|
|
||||||
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
|
||||||
assert.match(users, /anchor-sonarr/, `the scoped account is not on the broker:\n${users}`);
|
|
||||||
});
|
|
||||||
@@ -29,7 +29,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -42,7 +42,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "tools-confluence";
|
const SCENARIO = "tools-confluence";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -155,35 +155,9 @@ test("the mesh assigns confluence: its tools-only runtime comes up and serves th
|
|||||||
// confluence is tools-only and outbound-only: no service, no listener, no provisioner — just a
|
// confluence is tools-only and outbound-only: no service, no listener, no provisioner — just a
|
||||||
// broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the
|
// broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the
|
||||||
// lab has no real Confluence, so the token points at nothing — and that is the case under test: the
|
// lab has no real Confluence, so the token points at nothing — and that is the case under test: the
|
||||||
// runtime must serve every tool regardless. The runtime container name and shape mirror the
|
// runtime must serve every tool regardless. The manifest is the catalogue's own, its runtime
|
||||||
// committed manifest, with the image pinned to what this scenario serves by digest.
|
// artifact the image this scenario stocked (novox/hq 04-ISSUES/073).
|
||||||
const manifest = JSON.stringify({
|
const manifest = catalogueModule("confluence", held);
|
||||||
module: "confluence",
|
|
||||||
version: "1",
|
|
||||||
"own-secrets": {
|
|
||||||
token: "/var/lib/confluence/token",
|
|
||||||
broker: "/var/lib/mesh/confluence/broker",
|
|
||||||
},
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/confluence", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/confluence", mode: "0700" },
|
|
||||||
{ id: "config", type: "file", path: "/var/lib/confluence/config.json", merge: "json", content: "{}", mode: "0600" },
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-runtime-confluence",
|
|
||||||
image: pinned("mesh-runtime-confluence"), network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/confluence/config.json:/run/config/config.json:ro",
|
|
||||||
"/var/lib/confluence/token:/run/secrets/token:ro",
|
|
||||||
"/var/lib/mesh/confluence/broker:/run/secrets/broker:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_CONFLUENCE_TOKEN_FILE: "/run/secrets/token",
|
|
||||||
MESH_CONFLUENCE_CONFIG_FILE: "/run/config/config.json",
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/confluence.json && docker cp /tmp/confluence.json mesh-controller:/confluence.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/confluence.json && docker cp /tmp/confluence.json mesh-controller:/confluence.json`);
|
||||||
await mesh("module add /confluence.json");
|
await mesh("module add /confluence.json");
|
||||||
@@ -230,6 +204,15 @@ test("the mesh assigns confluence: its tools-only runtime comes up and serves th
|
|||||||
assert.match(served2, /serve\.confluence\.confluence_search/,
|
assert.match(served2, /serve\.confluence\.confluence_search/,
|
||||||
`confluence's runtime never bound its serve queue:\n${(await on(`docker logs mesh-runtime-confluence 2>&1 | tail -20`)).out}\n---\n${served2}`);
|
`confluence's runtime never bound its serve queue:\n${(await on(`docker logs mesh-runtime-confluence 2>&1 | tail -20`)).out}\n---\n${served2}`);
|
||||||
|
|
||||||
|
// --- and the control plane is the way to ask it (novox/hq ADR 0095, issue 049) ------------------
|
||||||
|
// No account in the mesh but the control plane's may create a reply queue and publish to a
|
||||||
|
// module's request key. Asked through it, the tool ANSWERS — with an error, since the lab has no
|
||||||
|
// Confluence and no token, which is an answer: the round trip is what is under test, and a
|
||||||
|
// timeout would read differently.
|
||||||
|
const asked = await on(`docker exec mesh-controller /mesh-controller ask confluence confluence_search '{"query":"mesh"}' --wait 60s`, 90_000);
|
||||||
|
assert.doesNotMatch(asked.out, /did not answer/, `the tool was never reached through the control plane:\n${asked.out}`);
|
||||||
|
assert.match(asked.out, /"(result|error)"/, `the control plane printed no answer:\n${asked.out}`);
|
||||||
|
|
||||||
// --- confluence got its own scoped broker account -----------------------------------------------
|
// --- confluence got its own scoped broker account -----------------------------------------------
|
||||||
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
const users = await must(`docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
||||||
assert.match(users, /anchor-confluence/, `the scoped account anchor-confluence is not on the broker:\n${users}`);
|
assert.match(users, /anchor-confluence/, `the scoped account anchor-confluence is not on the broker:\n${users}`);
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -41,7 +41,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "tools-gitlab";
|
const SCENARIO = "tools-gitlab";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -154,35 +154,9 @@ test("the mesh assigns gitlab: its tools-only runtime comes up and serves the fu
|
|||||||
// gitlab is tools-only and outbound-only: no service, no listener, no provisioner — just a
|
// gitlab is tools-only and outbound-only: no service, no listener, no provisioner — just a
|
||||||
// broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the
|
// broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the
|
||||||
// lab has no real GitLab, so the token points at nothing — and that is the case under test: the
|
// lab has no real GitLab, so the token points at nothing — and that is the case under test: the
|
||||||
// runtime must serve every tool regardless. The runtime container name and shape mirror the
|
// runtime must serve every tool regardless. The manifest is the catalogue's own, its runtime
|
||||||
// committed manifest, with the image pinned to what this scenario serves by digest.
|
// artifact the image this scenario stocked (novox/hq 04-ISSUES/073).
|
||||||
const manifest = JSON.stringify({
|
const manifest = catalogueModule("gitlab", held);
|
||||||
module: "gitlab",
|
|
||||||
version: "1",
|
|
||||||
"own-secrets": {
|
|
||||||
token: "/var/lib/gitlab/token",
|
|
||||||
broker: "/var/lib/mesh/gitlab/broker",
|
|
||||||
},
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/gitlab", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/gitlab", mode: "0700" },
|
|
||||||
{ id: "config", type: "file", path: "/var/lib/gitlab/config.json", merge: "json", content: "{}", mode: "0600" },
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-runtime-gitlab",
|
|
||||||
image: pinned("mesh-runtime-gitlab"), network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/gitlab/config.json:/run/config/config.json:ro",
|
|
||||||
"/var/lib/gitlab/token:/run/secrets/token:ro",
|
|
||||||
"/var/lib/mesh/gitlab/broker:/run/secrets/broker:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_GITLAB_TOKEN_FILE: "/run/secrets/token",
|
|
||||||
MESH_GITLAB_CONFIG_FILE: "/run/config/config.json",
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/gitlab.json && docker cp /tmp/gitlab.json mesh-controller:/gitlab.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/gitlab.json && docker cp /tmp/gitlab.json mesh-controller:/gitlab.json`);
|
||||||
await mesh("module add /gitlab.json");
|
await mesh("module add /gitlab.json");
|
||||||
|
|||||||
@@ -8,15 +8,14 @@
|
|||||||
*
|
*
|
||||||
* This bed proves that across two machines. `anchor` is the control-node: it raises the foundation
|
* This bed proves that across two machines. `anchor` is the control-node: it raises the foundation
|
||||||
* and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner.
|
* and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner.
|
||||||
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database` — plus
|
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database`. Each
|
||||||
* a co-located `redis` (which holds no seat). Each consumer's database is minted on the store on
|
* consumer's database is minted on the store on anchor and reached over the overlay: their bindings
|
||||||
* anchor and reached over the overlay: their bindings name `anchor.internal`, and their minted logins
|
* name `anchor.internal`, and their minted logins authenticate against the store. baserow's cache is
|
||||||
* authenticate against the store. redis stays co-located on laptop for baserow's cache.
|
* its own, inside its container (novox/hq 081).
|
||||||
*
|
*
|
||||||
* The four manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
|
* The three manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
|
||||||
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, redis runs on
|
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, and baserow/letta
|
||||||
* the host network with a lab-local seal key, and baserow/letta wire their servers to the grant the
|
* wire their servers to the grant the mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
|
||||||
* mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
|
|
||||||
* ONCE; laptop converges once with every one up, and the two consumers are provisioned against the
|
* ONCE; laptop converges once with every one up, and the two consumers are provisioned against the
|
||||||
* database the provider on their own node gave them.
|
* database the provider on their own node gave them.
|
||||||
*
|
*
|
||||||
@@ -25,25 +24,23 @@
|
|||||||
* MESH_LAB_HOST_BINARY=.../mesh-host
|
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||||
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||||
*
|
*
|
||||||
* HELPER — stock the four runtimes into the local daemon before the run (some may already be there):
|
* HELPER — stock the three runtimes into the local daemon before the run (some may already be there):
|
||||||
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar
|
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar
|
||||||
* scripts/build-module-runtime.sh redis /tmp/redis.tar
|
|
||||||
* scripts/build-module-runtime.sh baserow /tmp/baserow.tar
|
* scripts/build-module-runtime.sh baserow /tmp/baserow.tar
|
||||||
* scripts/build-module-runtime.sh letta /tmp/letta.tar
|
* scripts/build-module-runtime.sh letta /tmp/letta.tar
|
||||||
* The service images (postgres:17-alpine, redis:7-alpine, baserow/baserow:latest, letta/letta:latest)
|
* The service images (postgres, baserow, letta, each pinned by digest)
|
||||||
* must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls
|
* must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls
|
||||||
* what it runs from the scenario's own registry by digest.
|
* what it runs from the scenario's own registry by digest.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
import { test, before, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { dirname, resolve } from "node:path";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -56,16 +53,13 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "two-node-db";
|
const SCENARIO = "two-node-db";
|
||||||
/** The node that carries the whole DB-consumer chain. anchor carries only the foundation. */
|
/** The node that carries the whole DB-consumer chain. anchor carries only the foundation. */
|
||||||
const NODE = "laptop";
|
const NODE = "laptop";
|
||||||
|
|
||||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
|
||||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
|
||||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
|
||||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
|
||||||
|
|
||||||
let instanceId = "";
|
let instanceId = "";
|
||||||
/** The mesh's own images, as the machines hold them. */
|
/** The mesh's own images, as the machines hold them. */
|
||||||
@@ -156,12 +150,13 @@ async function settled(node: string, withinMs = 1_200_000): Promise<void> {
|
|||||||
last = said;
|
last = said;
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
await new Promise((r) => setTimeout(r, 5000));
|
||||||
}
|
}
|
||||||
// Timed out — capture what the node is actually doing so the failure is diagnosable.
|
// Timed out — capture what the node that did not answer is doing, so the failure is diagnosable.
|
||||||
const ps = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
// Its own machine, not the second node's: the anchor timing out used to print the laptop's log.
|
||||||
const hostLog = (await on(NODE, `tail -80 /var/log/mesh-host.log`)).out;
|
const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
|
||||||
|
const hostLog = (await on(node, `tail -80 /var/log/mesh-host.log`)).out;
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`${node} never caught up within ${Math.round(withinMs / 1000)}s.\nLast status:\n${last}\n` +
|
`${node} never caught up within ${Math.round(withinMs / 1000)}s.\nLast status:\n${last}\n` +
|
||||||
`--- ${NODE} docker ps -a ---\n${ps}\n--- ${NODE} mesh-host.log tail ---\n${hostLog}`);
|
`--- ${node} docker ps -a ---\n${ps}\n--- ${node} mesh-host.log tail ---\n${hostLog}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
@@ -209,82 +204,38 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
// they land on changes.
|
// they land on changes.
|
||||||
// ================================================================================================
|
// ================================================================================================
|
||||||
|
|
||||||
// --- redis: a cache provider on the host network (127.0.0.1:6379). No seal key: the provider
|
// --- baserow: a consumer that requires postgres-database, its server wired to the grant the mesh
|
||||||
// is handed the minted credential already unsealed by the host (ADR 0048). It carries
|
// writes, plus a runtime that serves baserow's tools. Its cache is its own — the image runs one when
|
||||||
// the committed provides/serves/receives/grants so baserow's redis-cache requirement resolves. ----
|
// no REDIS_HOST is given — because baserow keeps keys and channels under fixed names a shared
|
||||||
const redisManifest = JSON.stringify({
|
// cache's per-consumer grant cannot confine (novox/hq 081). --------------------------------------
|
||||||
module: "redis",
|
|
||||||
version: "1",
|
|
||||||
provides: [{ name: "redis-cache", scope: "mesh" }],
|
|
||||||
serves: { "redis-cache": { port: 6379 } },
|
|
||||||
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
|
|
||||||
grants: { "redis-cache": "/var/lib/redis-module/grants" },
|
|
||||||
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
|
|
||||||
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
|
|
||||||
{
|
|
||||||
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
|
|
||||||
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/services/redis/data:/data",
|
|
||||||
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
|
|
||||||
],
|
|
||||||
args: ["/etc/redis/redis.conf"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
|
|
||||||
network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
|
|
||||||
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
|
|
||||||
GRANTS: "/var/lib/redis-module/grants",
|
|
||||||
MESH_PROVISION_REDIS: "127.0.0.1:6379",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
// --- baserow: a consumer that requires postgres-database AND redis-cache; server wired to both
|
|
||||||
// from the grants the mesh writes, plus a runtime that serves baserow's tools. --------------------
|
|
||||||
const baserowManifest = JSON.stringify({
|
const baserowManifest = JSON.stringify({
|
||||||
module: "baserow",
|
module: "baserow",
|
||||||
version: "1",
|
version: "1",
|
||||||
requires: ["postgres-database", "redis-cache"],
|
requires: ["postgres-database"],
|
||||||
contributes: { "postgres-database": { name: "baserow" } },
|
contributes: { "postgres-database": { name: "baserow" } },
|
||||||
binds: { "postgres-database": "/var/lib/baserow/database.json", "redis-cache": "/var/lib/baserow/redis.json" },
|
binds: { "postgres-database": "/var/lib/baserow/database.json" },
|
||||||
secrets: { "postgres-database": "/var/lib/baserow/database.secret", "redis-cache": "/var/lib/baserow/redis.secret" },
|
secrets: { "postgres-database": "/var/lib/baserow/database.secret" },
|
||||||
"own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" },
|
"own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" },
|
||||||
resources: [
|
resources: [
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" },
|
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" },
|
||||||
{ id: "state", type: "directory", path: "/var/lib/baserow", mode: "0700" },
|
{ id: "state", type: "directory", path: "/var/lib/baserow", mode: "0700" },
|
||||||
{ id: "data", type: "directory", path: "/services/baserow/data", mode: "0700", owner: "9999:9999" },
|
// 0755, as the image ships it: the cache it runs for itself does so as another user, who
|
||||||
|
// must be able to reach its own directory under this one (novox/hq 081).
|
||||||
|
{ id: "data", type: "directory", path: "/services/baserow/data", mode: "0755", owner: "9999:9999" },
|
||||||
{
|
{
|
||||||
id: "server-env", type: "file", path: "/var/lib/baserow/server.env", mode: "0600",
|
id: "server-env", type: "file", path: "/var/lib/baserow/server.env", mode: "0600",
|
||||||
content:
|
content:
|
||||||
"DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" +
|
"DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" +
|
||||||
"DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" +
|
"DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" +
|
||||||
"DATABASE_PASSWORD=${secret:postgres-database}\nREDIS_HOST=${bound:redis-cache:at}\n" +
|
"DATABASE_PASSWORD=${secret:postgres-database}\n" +
|
||||||
"REDIS_PORT=${bound:redis-cache:port}\nREDIS_PROTOCOL=redis\nREDIS_PASSWORD=${secret:redis-cache}\n" +
|
|
||||||
"SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n",
|
"SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n",
|
||||||
},
|
},
|
||||||
{ id: "net", type: "network", name: "baserow" },
|
{ id: "net", type: "network", name: "baserow" },
|
||||||
{
|
{
|
||||||
id: "server", type: "container", name: "baserow", image: pinned("baserow/baserow"), network: "baserow",
|
id: "server", type: "container", name: "baserow", image: pinned("baserow/baserow"), network: "baserow",
|
||||||
"env-file": ["/var/lib/baserow/server.env"],
|
"env-file": ["/var/lib/baserow/server.env"],
|
||||||
|
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
|
||||||
|
"secrets-in-environment": "baserow reads DATABASE_PASSWORD and SECRET_KEY with os.getenv and has no _FILE twin (settings/base.py); not convertible",
|
||||||
volumes: ["/services/baserow/data:/baserow/data"],
|
volumes: ["/services/baserow/data:/baserow/data"],
|
||||||
},
|
},
|
||||||
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/baserow/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/baserow/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
||||||
@@ -329,6 +280,8 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
{
|
{
|
||||||
id: "server", type: "container", name: "letta", image: pinned("letta/letta"), network: "letta",
|
id: "server", type: "container", name: "letta", image: pinned("letta/letta"), network: "letta",
|
||||||
"env-file": ["/var/lib/letta/server.env"],
|
"env-file": ["/var/lib/letta/server.env"],
|
||||||
|
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
|
||||||
|
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
|
||||||
},
|
},
|
||||||
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/letta/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
{ id: "runtime-config", type: "file", path: "/var/lib/mesh/letta/config.json", mode: "0600", content: "{}\n", merge: "json" },
|
||||||
{ id: "runtime-env", type: "file", path: "/var/lib/letta/runtime.env", mode: "0600", content: "MESH_LETTA_PASSWORD=${secret:server-password}\n" },
|
{ id: "runtime-env", type: "file", path: "/var/lib/letta/runtime.env", mode: "0600", content: "MESH_LETTA_PASSWORD=${secret:server-password}\n" },
|
||||||
@@ -345,6 +298,8 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
MESH_LETTA_CONFIG_FILE: "/run/config/config.json",
|
MESH_LETTA_CONFIG_FILE: "/run/config/config.json",
|
||||||
},
|
},
|
||||||
"env-file": ["/var/lib/letta/runtime.env"],
|
"env-file": ["/var/lib/letta/runtime.env"],
|
||||||
|
// Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
|
||||||
|
"secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
|
||||||
"restart-on": ["runtime-config"],
|
"restart-on": ["runtime-config"],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -359,22 +314,11 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
await mesh(`assign ${NODE} ${name}`);
|
await mesh(`assign ${NODE} ${name}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load a committed catalog module.json with its container images rewritten to this scenario's
|
// The catalogue's manifest as the lab runs it (harness), so the foundation store can be ADOPTED
|
||||||
// pinned digests, so the foundation store can be ADOPTED in place as the one postgres.
|
// in place as the one postgres.
|
||||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||||
const m = JSON.parse(readFileSync(resolve(catalogDir, name, "module.json"), "utf8")) as {
|
const manifest = catalogueModule(name, held);
|
||||||
resources?: { type: string; image?: string; artifact?: string }[];
|
return { manifest, broker: needsBrokerAccount(manifest) };
|
||||||
};
|
|
||||||
for (const r of m.resources ?? []) {
|
|
||||||
if (r.type !== "container") continue;
|
|
||||||
if (typeof r.image === "string") r.image = pinned(r.image);
|
|
||||||
else if (typeof r.artifact === "string") {
|
|
||||||
r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`);
|
|
||||||
delete r.artifact;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const manifest = JSON.stringify(m);
|
|
||||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
|
||||||
}
|
}
|
||||||
async function installCatalog(name: string, node: string): Promise<void> {
|
async function installCatalog(name: string, node: string): Promise<void> {
|
||||||
const { manifest, broker } = loadManifest(name);
|
const { manifest, broker } = loadManifest(name);
|
||||||
@@ -416,8 +360,7 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
await mesh(`push anchor`, 600_000);
|
await mesh(`push anchor`, 600_000);
|
||||||
await settled("anchor");
|
await settled("anchor");
|
||||||
|
|
||||||
// The consumers ride laptop; redis is an ordinary co-located provider (it holds no seat).
|
// The consumers ride laptop.
|
||||||
await addIssueAssign("redis", redisManifest);
|
|
||||||
await addIssueAssign("baserow", baserowManifest);
|
await addIssueAssign("baserow", baserowManifest);
|
||||||
await addIssueAssign("letta", lettaManifest);
|
await addIssueAssign("letta", lettaManifest);
|
||||||
await mesh(`push ${NODE}`);
|
await mesh(`push ${NODE}`);
|
||||||
@@ -444,7 +387,6 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
// THE co-residence proof — every module's containers up and stable on the second node.
|
// THE co-residence proof — every module's containers up and stable on the second node.
|
||||||
// ================================================================================================
|
// ================================================================================================
|
||||||
const expected = [
|
const expected = [
|
||||||
"redis", "mesh-redis",
|
|
||||||
"baserow", "mesh-baserow",
|
"baserow", "mesh-baserow",
|
||||||
"letta", "mesh-letta",
|
"letta", "mesh-letta",
|
||||||
];
|
];
|
||||||
@@ -520,7 +462,7 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
// reached from laptop over the shared segment) — named for the node that runs it and the module.
|
// reached from laptop over the shared segment) — named for the node that runs it and the module.
|
||||||
// ================================================================================================
|
// ================================================================================================
|
||||||
const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`);
|
||||||
for (const acct of ["anchor-postgres", "laptop-redis", "laptop-baserow", "laptop-letta"]) {
|
for (const acct of ["anchor-postgres", "laptop-baserow", "laptop-letta"]) {
|
||||||
assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`);
|
assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -552,14 +494,26 @@ test("consumers on a joined node get their databases from the one foundation sto
|
|||||||
assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`);
|
assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// baserow also got its redis-cache binding: the mesh wrote the binding and unsealed the secret,
|
// baserow's cache is its own: with no REDIS_HOST the image runs one inside the container, under a
|
||||||
// and both arrived on the second node (a live redis AUTH is left to the redis single-module bed
|
// password it makes itself, and the mesh grants nothing (novox/hq 081). Three things say so:
|
||||||
// and the open provider-seal-key work).
|
// baserow said it chose its own; the cache answers on loopback with the challenge for that password
|
||||||
const redisBound = await waitForBinding("/var/lib/baserow/redis.json");
|
// (PONG would be a cache anyone can use, and fails); and nothing was refused a login.
|
||||||
assert.equal(redisBound.provision, "redis-cache", `baserow's redis binding is the wrong provision: ${redisBound.provision}`);
|
const baserowLog = async () => (await on(NODE, `docker logs baserow 2>&1`)).out;
|
||||||
assert.ok(redisBound.as, `baserow's redis binding carries no login:\n${JSON.stringify(redisBound)}`);
|
let chose = "";
|
||||||
const redisSecret = (await must(NODE, `cat /var/lib/baserow/redis.secret`)).trim();
|
let cache = { out: "", ok: false };
|
||||||
assert.ok(redisSecret.length > 0, "baserow's redis secret was not delivered");
|
const untilCache = Date.now() + 240_000;
|
||||||
|
while (Date.now() < untilCache) {
|
||||||
|
chose = await baserowLog();
|
||||||
|
cache = await on(NODE, `docker exec baserow redis-cli -h 127.0.0.1 ping 2>&1`);
|
||||||
|
if (/Using embedded baserow redis/.test(chose) && /NOAUTH/.test(cache.out)) break;
|
||||||
|
await new Promise((r) => setTimeout(r, 5000));
|
||||||
|
}
|
||||||
|
assert.match(chose, /Using embedded baserow redis/,
|
||||||
|
`baserow did not start its own cache:\n${chose.split("\n").filter((l) => /redis/i.test(l)).slice(-15).join("\n")}`);
|
||||||
|
assert.match(cache.out, /NOAUTH/,
|
||||||
|
`baserow's own cache does not answer with its password challenge inside the container:\n${cache.out}`);
|
||||||
|
assert.doesNotMatch(chose, /WRONGPASS|NOPERM/,
|
||||||
|
`baserow was refused by its cache:\n${chose.split("\n").filter((l) => /WRONGPASS|NOPERM/.test(l)).slice(-15).join("\n")}`);
|
||||||
|
|
||||||
// Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it.
|
// Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it.
|
||||||
async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {
|
async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {
|
||||||
|
|||||||
@@ -259,9 +259,32 @@ test("redis's own password is a secret the vault provides: it authenticates, and
|
|||||||
assert.match(issued, /scoped to what it emits and consumes/, issued);
|
assert.match(issued, /scoped to what it emits and consumes/, issued);
|
||||||
await mesh(`assign ${MACHINE} ${name}`);
|
await mesh(`assign ${MACHINE} ${name}`);
|
||||||
}
|
}
|
||||||
|
// A consumer that needs TWO values from the vault (novox/hq ADR 0094): its `secrets` entry names
|
||||||
|
// them under local names, and each is a pair of its own. It runs no code — the delivery is what
|
||||||
|
// is under test. Synthetic, so it wears no catalogue module's name.
|
||||||
|
const twoSecrets = JSON.stringify({
|
||||||
|
module: "two-secrets", version: "1", slug: "two",
|
||||||
|
requires: ["secret"],
|
||||||
|
secrets: { secret: { first: "/var/lib/two-secrets/first", second: "/var/lib/two-secrets/second" } },
|
||||||
|
resources: [{ id: "state", type: "directory", path: "/var/lib/two-secrets", mode: "0700" }],
|
||||||
|
});
|
||||||
|
await must(`printf %s ${quote(twoSecrets)} > /tmp/two-secrets.json && docker cp /tmp/two-secrets.json mesh-controller:/two-secrets.json`);
|
||||||
|
await mesh("module add /two-secrets.json");
|
||||||
|
await mesh(`assign ${MACHINE} two-secrets`);
|
||||||
await mesh(`push ${MACHINE}`);
|
await mesh(`push ${MACHINE}`);
|
||||||
await settled();
|
await settled();
|
||||||
|
|
||||||
|
// Two files, two values, and the vault holds two holders for one module — the identity with the
|
||||||
|
// local name after it.
|
||||||
|
const first = (await must(`cat /var/lib/two-secrets/first`)).replace(/\n$/, "");
|
||||||
|
const second = (await must(`cat /var/lib/two-secrets/second`)).replace(/\n$/, "");
|
||||||
|
assert.ok(first.length >= 20 && second.length >= 20, "a two-secrets value is empty or implausibly short");
|
||||||
|
assert.notEqual(first, second, "two local names were given one value");
|
||||||
|
for (const holderOf of ["mesh_anchor_two_first", "mesh_anchor_two_second"]) {
|
||||||
|
await until(`the vault holding ${holderOf}`, 90_000, async () =>
|
||||||
|
(await on(`test -s ${LEDGER}/${holderOf}.json`)).ok ? true : undefined);
|
||||||
|
}
|
||||||
|
|
||||||
const running = await must(`docker ps --format '{{.Names}}'`);
|
const running = await must(`docker ps --format '{{.Names}}'`);
|
||||||
for (const c of ["mesh-vault", "redis", "mesh-redis"]) {
|
for (const c of ["mesh-vault", "redis", "mesh-redis"]) {
|
||||||
assert.match(running, new RegExp(`(^|\\n)${c}(\\n|$)`),
|
assert.match(running, new RegExp(`(^|\\n)${c}(\\n|$)`),
|
||||||
@@ -335,6 +358,21 @@ test("rotating the secret moves both ends: the new password works, the old one i
|
|||||||
});
|
});
|
||||||
assert.notEqual(after, before);
|
assert.notEqual(after, before);
|
||||||
|
|
||||||
|
// Both of the two-secrets consumer's values moved too, apart from each other (ADR 0094).
|
||||||
|
const firstAfter = await until("the rotated first secret", 180_000, async () => {
|
||||||
|
const now = (await must(`cat /var/lib/two-secrets/first`)).replace(/\n$/, "");
|
||||||
|
return now !== "" ? now : undefined;
|
||||||
|
});
|
||||||
|
const secondAfter = (await must(`cat /var/lib/two-secrets/second`)).replace(/\n$/, "");
|
||||||
|
assert.notEqual(firstAfter, secondAfter, "two local names were given one value after rotation");
|
||||||
|
for (const holderOf of ["mesh_anchor_two_first", "mesh_anchor_two_second"]) {
|
||||||
|
const h = await until(`the vault recording ${holderOf}'s rotation`, 90_000, async () => {
|
||||||
|
const got = JSON.parse(await must(`cat ${LEDGER}/${holderOf}.json`)) as Held;
|
||||||
|
return got.rotations >= 1 ? got : undefined;
|
||||||
|
});
|
||||||
|
assert.equal(h.rotations, 1, holderOf);
|
||||||
|
}
|
||||||
|
|
||||||
// Three logins. The new one works (redis was restarted on its config — `restart-on`), the old one
|
// Three logins. The new one works (redis was restarted on its config — `restart-on`), the old one
|
||||||
// does not: that third check is what makes it a rotation rather than an addition.
|
// does not: that third check is what makes it a rotation rather than an addition.
|
||||||
await until("redis accepting the rotated password", 180_000, async () => {
|
await until("redis accepting the rotated password", 180_000, async () => {
|
||||||
|
|||||||
@@ -41,6 +41,14 @@ const ACME = "/var/lib/acme";
|
|||||||
*/
|
*/
|
||||||
const AUTHORITY = "ghcr.io/letsencrypt/pebble:2.5.0";
|
const AUTHORITY = "ghcr.io/letsencrypt/pebble:2.5.0";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The second implementation, for the same order (novox/hq 04-ISSUES/020): the certificate
|
||||||
|
* authority the catalogue itself runs, pinned as the catalogue pins it. If the order, the challenge
|
||||||
|
* and the handshake agree here as well as against Pebble, the one thing 020 could not rule out — a
|
||||||
|
* Pebble interop detail — is ruled out; and if they disagree, which side differs is in view.
|
||||||
|
*/
|
||||||
|
const SECOND_AUTHORITY = "smallstep/step-ca@sha256:a2b17872915c193259b75a5474c398326f41bd199f0842093e52cf4182bc8270";
|
||||||
|
|
||||||
let instanceId = "";
|
let instanceId = "";
|
||||||
|
|
||||||
function shellQuote(s: string): string {
|
function shellQuote(s: string): string {
|
||||||
@@ -127,9 +135,18 @@ before(async () => {
|
|||||||
given: [{ from: "photos", node: "", at: "", values: { name: NAME, port: 8080 } }],
|
given: [{ from: "photos", node: "", at: "", values: { name: NAME, port: 8080 } }],
|
||||||
}))} > ${ACME}/routes.json`,
|
}))} > ${ACME}/routes.json`,
|
||||||
);
|
);
|
||||||
await must(
|
// A real small server, not a netcat loop: the loop's `nc -l -p … -q` is not this machine's netcat,
|
||||||
`nohup sh -c 'while true; do printf "HTTP/1.1 200 OK\\r\\nContent-Length: 5\\r\\n\\r\\nhello" | nc -l -p 8080 -q 1; done' >/dev/null 2>&1 &`,
|
// so it never listened, and every request through the proxy was refused by the backend — which
|
||||||
);
|
// read as the certificate never arriving, and hid behind the authority's refusal until the
|
||||||
|
// second authority issued one.
|
||||||
|
await must(`mkdir -p ${ACME}/www && printf hello > ${ACME}/www/index.html`);
|
||||||
|
await must(`nohup python3 -m http.server 8080 --bind 127.0.0.1 --directory ${ACME}/www >${ACME}/backend.log 2>&1 &`);
|
||||||
|
let backend = false;
|
||||||
|
for (let i = 0; i < 20 && !backend; i++) {
|
||||||
|
({ ok: backend } = await on(`curl -sf http://127.0.0.1:8080/ -o /dev/null`));
|
||||||
|
if (!backend) await new Promise((r) => setTimeout(r, 1000));
|
||||||
|
}
|
||||||
|
assert.ok(backend, `the backend behind the route never answered:\n${(await on(`cat ${ACME}/backend.log`)).out}`);
|
||||||
}, { timeout: 1_200_000 });
|
}, { timeout: 1_200_000 });
|
||||||
|
|
||||||
after(async () => {
|
after(async () => {
|
||||||
@@ -177,12 +194,69 @@ test("a public name is served with a certificate the mesh did not issue", {
|
|||||||
assert.match(served.out, /hello/);
|
assert.match(served.out, /hello/);
|
||||||
|
|
||||||
// And it is the authority's certificate, not something self-signed that happens to work.
|
// And it is the authority's certificate, not something self-signed that happens to work.
|
||||||
const issuer = await must(
|
// The issuer, and the names the certificate is FOR — its alternative names, not its subject:
|
||||||
|
// Pebble, like the public authority it stands in for, leaves the subject empty and puts the
|
||||||
|
// name in the alternative names alone. The first version of this read the subject and refused
|
||||||
|
// a valid certificate.
|
||||||
|
const issued = await must(
|
||||||
`echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` +
|
`echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` +
|
||||||
`| openssl x509 -noout -issuer -subject`,
|
`| openssl x509 -noout -issuer -ext subjectAltName`,
|
||||||
);
|
);
|
||||||
assert.match(issuer, /Pebble/i, `the certificate was not issued by the ACME server:\n${issuer}`);
|
assert.match(issued, /Pebble/i, `the certificate was not issued by the ACME server:\n${issued}`);
|
||||||
assert.match(issuer, new RegExp(NAME), `the certificate is not for the name asked for:\n${issuer}`);
|
assert.match(issued, new RegExp(`DNS:${NAME.replace(".", "\\.")}`), `the certificate is not for the name asked for:\n${issued}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the same order against a second authority: the catalogue's own certificate authority", {
|
||||||
|
skip, timeout: 900_000,
|
||||||
|
}, async () => {
|
||||||
|
// The proxy that served the first test goes; its cache with it, or the certificate Pebble issued
|
||||||
|
// would be served again and nothing would have been ordered here.
|
||||||
|
await must(`pkill -f '^/usr/local/bin/mesh-route-proxy' || true; sleep 1; mkdir -p ${ACME}/cache2`);
|
||||||
|
|
||||||
|
// The catalogue's authority, as the catalogue runs it: ACME on, listening on its own port, a
|
||||||
|
// root and an intermediate made at first start. It resolves the name through the machine's
|
||||||
|
// resolver, which reads the hosts entry the first test wrote.
|
||||||
|
await must(
|
||||||
|
`docker run -d --name stepca --network host ` +
|
||||||
|
`-e DOCKER_STEPCA_INIT_NAME="Lab CA" -e DOCKER_STEPCA_INIT_DNS_NAMES=localhost,127.0.0.1 ` +
|
||||||
|
`-e DOCKER_STEPCA_INIT_ACME=true -e DOCKER_STEPCA_INIT_REMOTE_MANAGEMENT=false ` +
|
||||||
|
`-e DOCKER_STEPCA_INIT_PASSWORD=lab-only-password ${SECOND_AUTHORITY}`,
|
||||||
|
);
|
||||||
|
let ready = false;
|
||||||
|
for (let i = 0; i < 90 && !ready; i++) {
|
||||||
|
({ ok: ready } = await on(`docker exec stepca test -s /home/step/certs/root_ca.crt && curl -sk https://127.0.0.1:9000/health -o /dev/null`));
|
||||||
|
if (!ready) await new Promise((r) => setTimeout(r, 2000));
|
||||||
|
}
|
||||||
|
assert.ok(ready, `the second authority never came up:\n${(await on(`docker logs stepca 2>&1 | tail -30`)).out}`);
|
||||||
|
await must(`docker exec stepca cat /home/step/certs/root_ca.crt > ${ACME}/stepca-root.pem`);
|
||||||
|
|
||||||
|
await must(
|
||||||
|
`ROUTES=${ACME}/routes.json LISTEN=:80 TLS_LISTEN=:443 ` +
|
||||||
|
`ACME_CACHE=${ACME}/cache2 ` +
|
||||||
|
`ACME_DIRECTORY=https://127.0.0.1:9000/acme/acme/directory ` +
|
||||||
|
`ACME_CA_BUNDLE=${ACME}/stepca-root.pem ` +
|
||||||
|
`nohup /usr/local/bin/mesh-route-proxy >${ACME}/proxy2.log 2>&1 & sleep 3`,
|
||||||
|
);
|
||||||
|
|
||||||
|
let served = { out: "", ok: false };
|
||||||
|
for (let i = 0; i < 40 && !served.ok; i++) {
|
||||||
|
served = await on(`curl -sf --cacert ${ACME}/stepca-root.pem https://${NAME}/ `);
|
||||||
|
if (!served.ok) await new Promise((r) => setTimeout(r, 2000));
|
||||||
|
}
|
||||||
|
if (!served.ok) {
|
||||||
|
const proxyLog = (await on(`cat ${ACME}/proxy2.log`)).out;
|
||||||
|
const authority = (await on(`docker logs stepca 2>&1 | tail -40`)).out;
|
||||||
|
assert.fail(
|
||||||
|
`the name was never served over TLS from the second authority: ${served.out}\n\n` +
|
||||||
|
`── the proxy tried:\n${proxyLog}\n── the authority heard:\n${authority}\n`);
|
||||||
|
}
|
||||||
|
assert.match(served.out, /hello/);
|
||||||
|
const issued = await must(
|
||||||
|
`echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` +
|
||||||
|
`| openssl x509 -noout -issuer -ext subjectAltName`,
|
||||||
|
);
|
||||||
|
assert.match(issued, /Lab CA/, `the certificate was not issued by the second authority:\n${issued}`);
|
||||||
|
assert.match(issued, new RegExp(`DNS:${NAME.replace(".", "\\.")}`), `the certificate is not for the name asked for:\n${issued}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("no certificate is ordered for a name the mesh does not route", {
|
test("no certificate is ordered for a name the mesh does not route", {
|
||||||
@@ -193,6 +267,6 @@ test("no certificate is ordered for a name the mesh does not route", {
|
|||||||
const { out } = await on(
|
const { out } = await on(
|
||||||
`echo | openssl s_client -connect 127.0.0.1:443 -servername nobody-asked-for-this.example 2>&1 | head -20`,
|
`echo | openssl s_client -connect 127.0.0.1:443 -servername nobody-asked-for-this.example 2>&1 | head -20`,
|
||||||
);
|
);
|
||||||
assert.doesNotMatch(out, /Pebble/i,
|
assert.doesNotMatch(out, /Pebble|Lab CA/i,
|
||||||
`a certificate was obtained for a name nothing routes here:\n${out}`);
|
`a certificate was obtained for a name nothing routes here:\n${out}`);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,227 @@
|
|||||||
|
/**
|
||||||
|
* SPIKE for novox/hq 04-ISSUES/066 — a partly applied declaration leaves a mixed state.
|
||||||
|
*
|
||||||
|
* The apply is deliberately not a transaction: every resource is attempted, every failure reported,
|
||||||
|
* and a failed gate stops what follows it (issue 011, ADR 0053). The question 066 asks is whether a
|
||||||
|
* pairing exists whose half-state is harmful. This bed makes one, on purpose, and RECORDS what the
|
||||||
|
* machine is observed doing in it — it is evidence for a decision, not a rule being enforced.
|
||||||
|
*
|
||||||
|
* The pair: a config file and a long-lived container that serves the file's content as it was when
|
||||||
|
* the container started, with a run-once gate between them that validates the file. First push:
|
||||||
|
* the file says "v1", the gate passes, the service serves v1. Second push: the file says "v2" and
|
||||||
|
* the gate is made to refuse it. The file is applied before the gate (declaration order), the gate
|
||||||
|
* fails, the service after it is left as it was — so the machine has v2 on disk and serves v1, and
|
||||||
|
* reports the push as failed. That is the mixed state. Whether it is harmful is what a person
|
||||||
|
* decides from this; whether a `together` grouping should exist is what the decision would say.
|
||||||
|
*
|
||||||
|
* When such a grouping lands, this bed is where it is proven: the assertions below flip.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { test, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
||||||
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
|
const capability = await labIsUsable();
|
||||||
|
const binary = hostBinaryPath();
|
||||||
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
|
|
||||||
|
const skip = !capability.usable
|
||||||
|
? `lab not usable: ${capability.why}`
|
||||||
|
: !binary || !existsSync(binary)
|
||||||
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
|
: !bundle || !existsSync(bundle)
|
||||||
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
|
: false;
|
||||||
|
|
||||||
|
const SCENARIO = "schedule-tick"; // a bare node, as the tick bed uses
|
||||||
|
const MACHINE = "anchor";
|
||||||
|
|
||||||
|
let instanceId = "";
|
||||||
|
/** The mesh's own images, as the machines hold them. */
|
||||||
|
let held: HeldImage[] = [];
|
||||||
|
|
||||||
|
function quote(s: string): string {
|
||||||
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||||
|
const { stdout } = await exec(instanceId, MACHINE, [
|
||||||
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||||
|
], timeoutMs);
|
||||||
|
const marker = stdout.lastIndexOf("__exit=");
|
||||||
|
if (marker < 0) return { out: stdout, ok: false };
|
||||||
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function must(command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
const { out, ok } = await on(command, timeoutMs);
|
||||||
|
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The control plane, a container on the node. */
|
||||||
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The reference a manifest should carry, once this scenario has been raised. */
|
||||||
|
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
||||||
|
function pinned(reference: string): string {
|
||||||
|
return onTheMachine(reference, held);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
|
||||||
|
function bundleFor(images: HeldImage[]): string {
|
||||||
|
return foundationBundle(bundle, images);
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenFrom(said: string): string {
|
||||||
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||||
|
assert.ok(found, `no token in:\n${said}`);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How many lines the tick has written so far — tolerant of the run log not existing yet.
|
||||||
|
*
|
||||||
|
* The scheduled container appends one line per fire with `date >> /data/runs.log`, and the data
|
||||||
|
* directory is mounted from /var/lib/schedtest on the machine, so counting newlines there counts
|
||||||
|
* fires. `wc -l` on an absent file is an error, so a missing file reads as 0 rather than throwing —
|
||||||
|
* which is exactly the pre-first-fire state.
|
||||||
|
*/
|
||||||
|
async function tickLines(): Promise<number> {
|
||||||
|
const { out } = await on(`wc -l < /var/lib/schedtest/runs.log 2>/dev/null || echo 0`);
|
||||||
|
const n = Number.parseInt(out.trim(), 10);
|
||||||
|
return Number.isFinite(n) ? n : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function settled(withinMs = 600_000): Promise<void> {
|
||||||
|
const until = Date.now() + withinMs;
|
||||||
|
let last = "";
|
||||||
|
while (Date.now() < until) {
|
||||||
|
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
||||||
|
if (asked.ok) {
|
||||||
|
try {
|
||||||
|
const state = JSON.parse(asked.out) as {
|
||||||
|
wrong: { node: string; outcome: string }[];
|
||||||
|
waiting: { node: string }[];
|
||||||
|
reported: { node: string; outcome: string; current: boolean }[];
|
||||||
|
};
|
||||||
|
const bad = state.wrong.find((w) => w.node === MACHINE);
|
||||||
|
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
||||||
|
const word = state.reported.find((r) => r.node === MACHINE);
|
||||||
|
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
||||||
|
last = asked.out;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
||||||
|
last = asked.out;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 5000));
|
||||||
|
}
|
||||||
|
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
if (skip) return;
|
||||||
|
|
||||||
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
|
});
|
||||||
|
instanceId = raised.instanceId;
|
||||||
|
held = raised.images;
|
||||||
|
|
||||||
|
// Raise the foundation — store, broker, control — from the bundle.
|
||||||
|
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
||||||
|
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
||||||
|
const up = await must(`docker ps --format '{{.Names}}'`);
|
||||||
|
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||||
|
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The node joins its own mesh, so it is a node the mesh can assign to, and start the host so it
|
||||||
|
// applies what it is pushed AND fires scheduled steps off its clock (the daemon holds one
|
||||||
|
// Scheduler for the life of the process — novox/hq ADR 0053).
|
||||||
|
await mesh(`node add ${MACHINE}`);
|
||||||
|
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
||||||
|
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
||||||
|
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||||
|
}, { timeout: 1_800_000 });
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
if (instanceId) await destroy(instanceId);
|
||||||
|
await destroyAll(`${SCENARIO}-`);
|
||||||
|
}, { timeout: 600_000 });
|
||||||
|
|
||||||
|
function coupled(content: string, gateAccepts: boolean): string {
|
||||||
|
return JSON.stringify({
|
||||||
|
module: "coupled", version: "1",
|
||||||
|
resources: [
|
||||||
|
{ id: "state", type: "directory", path: "/var/lib/coupled", mode: "0755" },
|
||||||
|
{ id: "config", type: "file", path: "/var/lib/coupled/config", mode: "0644", content: content + "\n" },
|
||||||
|
// The gate: validates the file. Made to pass or fail from the manifest, which is the whole
|
||||||
|
// point — a real validator refusing a real bad config is exactly this shape.
|
||||||
|
{
|
||||||
|
id: "validate", type: "container", name: "coupled-validate", image: pinned("alpine"), "run-once": true,
|
||||||
|
volumes: ["/var/lib/coupled:/data:ro"],
|
||||||
|
args: ["sh", "-c", gateAccepts ? "test -s /data/config" : "echo 'config refused by the validator' >&2; exit 1"],
|
||||||
|
},
|
||||||
|
// The service: reads the file ONCE at start and serves that for its life, the way most
|
||||||
|
// servers read their configuration.
|
||||||
|
{
|
||||||
|
id: "service", type: "container", name: "coupled-service", image: pinned("alpine"), network: "host",
|
||||||
|
volumes: ["/var/lib/coupled:/data:ro"],
|
||||||
|
args: ["sh", "-c", "v=$(cat /data/config); while true; do printf 'HTTP/1.1 200 OK\\r\\nContent-Length: %s\\r\\n\\r\\n%s' \"${#v}\" \"$v\" | nc -l -p 8099; done"],
|
||||||
|
"restart-on": ["config"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function served(): Promise<string> {
|
||||||
|
return (await on(`curl -s --max-time 3 http://127.0.0.1:8099/ || true`)).out.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
test("a coupled pair half-applied: the file moved, the gate refused, the service serves the old file — the machine is observed in the mixed state", {
|
||||||
|
skip, timeout: 900_000,
|
||||||
|
}, async () => {
|
||||||
|
await must(`printf %s ${quote(coupled("v1", true))} > /tmp/coupled.json && docker cp /tmp/coupled.json mesh-controller:/coupled.json`);
|
||||||
|
await mesh("module add /coupled.json");
|
||||||
|
await mesh(`assign ${MACHINE} coupled`);
|
||||||
|
await mesh(`push ${MACHINE}`);
|
||||||
|
await settled();
|
||||||
|
let first = "";
|
||||||
|
for (let i = 0; i < 20 && first !== "v1"; i++) {
|
||||||
|
first = await served();
|
||||||
|
if (first !== "v1") await new Promise((r) => setTimeout(r, 2000));
|
||||||
|
}
|
||||||
|
assert.equal(first, "v1", "the service does not serve the first config");
|
||||||
|
|
||||||
|
// The change: a new file the validator refuses. Registered again under the same name so the
|
||||||
|
// mesh sends a new declaration; the file is applied, the gate fails, the service stays.
|
||||||
|
await must(`printf %s ${quote(coupled("v2", false))} > /tmp/coupled.json && docker cp /tmp/coupled.json mesh-controller:/coupled.json`);
|
||||||
|
await mesh("module add /coupled.json");
|
||||||
|
const pushed = await on(`docker exec mesh-controller /mesh-controller push ${MACHINE}`);
|
||||||
|
// The push is sent; what the machine did with it is read from its report.
|
||||||
|
let report = "";
|
||||||
|
for (let i = 0; i < 30; i++) {
|
||||||
|
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
||||||
|
report = asked.out;
|
||||||
|
if (/"outcome":\s*"failed"/.test(report)) break;
|
||||||
|
await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
}
|
||||||
|
assert.match(report, /"outcome":\s*"failed"/, `the machine did not report a failed apply:\n${pushed.out}\n${report}`);
|
||||||
|
|
||||||
|
// THE OBSERVATION. The file on disk is the new one; the service still serves the old one.
|
||||||
|
const onDisk = (await must(`cat /var/lib/coupled/config`)).trim();
|
||||||
|
const answered = await served();
|
||||||
|
assert.equal(onDisk, "v2", "the file was not applied before the gate");
|
||||||
|
assert.equal(answered, "v1", `the service was restarted onto a config the validator refused: ${answered}`);
|
||||||
|
console.log(`OBSERVED: config on disk = ${onDisk}, service serves = ${answered}, report = failed — the mixed state of novox/hq 04-ISSUES/066`);
|
||||||
|
});
|
||||||
@@ -93,6 +93,7 @@ before(async () => {
|
|||||||
step: "getting the machine ready to be bootstrapped — the installer never ran",
|
step: "getting the machine ready to be bootstrapped — the installer never ran",
|
||||||
why: (err as Error).message,
|
why: (err as Error).message,
|
||||||
report: [],
|
report: [],
|
||||||
|
said: "",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
console.log(result.report.join("\n"));
|
console.log(result.report.join("\n"));
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ export interface GenesisResult {
|
|||||||
step: string;
|
step: string;
|
||||||
why: string;
|
why: string;
|
||||||
report: string[];
|
report: string[];
|
||||||
|
/** Everything the installer printed on its last attempt — what a bed asserts a refusal names. */
|
||||||
|
said: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface GenesisOptions {
|
export interface GenesisOptions {
|
||||||
@@ -80,6 +82,23 @@ export interface GenesisOptions {
|
|||||||
hostBinary?: string;
|
hostBinary?: string;
|
||||||
/** What of the catalogue to build. A branch under test is the usual reason this is not main. */
|
/** What of the catalogue to build. A branch under test is the usual reason this is not main. */
|
||||||
catalogRef?: string;
|
catalogRef?: string;
|
||||||
|
/**
|
||||||
|
* Raise the machine adopted (novox/hq ADR 0100): what it runs and its firewall are kept. Without
|
||||||
|
* it the installer raises a converged node, and refuses a machine in use.
|
||||||
|
*/
|
||||||
|
adopted?: boolean;
|
||||||
|
/** Further installer flags, as the operator would type them — `--registry-port 5100`, `--dry-run`. */
|
||||||
|
flags?: string[];
|
||||||
|
/**
|
||||||
|
* How many times to run the installer. Three by default, for a pull the internet rate-limited; a
|
||||||
|
* bed that expects a REFUSAL runs it once, because a refusal is the answer, not a flake.
|
||||||
|
*/
|
||||||
|
attempts?: number;
|
||||||
|
/**
|
||||||
|
* Whether to ask the machine if it became a working mesh of one afterwards. Off for a run that is
|
||||||
|
* not meant to raise one — a dry run, or a refusal the bed expects.
|
||||||
|
*/
|
||||||
|
verify?: boolean;
|
||||||
log?: (m: string) => void;
|
log?: (m: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,9 +117,10 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
|||||||
const log = o.log ?? (() => {});
|
const log = o.log ?? (() => {});
|
||||||
|
|
||||||
const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`];
|
const report: string[] = [`================ GENESIS: ${node} becomes a mesh of one ================`];
|
||||||
|
let said = "";
|
||||||
const stop = (step: string, why: string): GenesisResult => {
|
const stop = (step: string, why: string): GenesisResult => {
|
||||||
report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
|
report.push(`\nSTOPPED at ${step || "(no step named)"}: ${why}`);
|
||||||
return { ok: false, step, why, report };
|
return { ok: false, step, why, report, said };
|
||||||
};
|
};
|
||||||
|
|
||||||
const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => {
|
const on = async (command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> => {
|
||||||
@@ -136,7 +156,9 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
|||||||
// the lab stands in for that by copying the whole tree once.
|
// the lab stands in for that by copying the whole tree once.
|
||||||
const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`);
|
const bundleTar = join(tmpdir(), `mesh-lab-catalogue-${process.pid}-${node}.tar`);
|
||||||
execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]);
|
execFileSync("tar", ["-cf", bundleTar, "-C", o.catalogDir, "."]);
|
||||||
await must(`mkdir -p ${catalogueOnMachine}/modules`);
|
// Cleared first: a bed that runs genesis more than once (a refusal, then the raise) finds the last
|
||||||
|
// run's staging files, and the machine refuses to open them for the push.
|
||||||
|
await must(`rm -f /tmp/catalogue.tar /tmp/foundation-template.lock && mkdir -p ${catalogueOnMachine}/modules`);
|
||||||
await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar");
|
await push(o.instanceId, node, bundleTar, "/tmp/catalogue.tar");
|
||||||
await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`);
|
await must(`tar -xf /tmp/catalogue.tar -C ${catalogueOnMachine}/modules`);
|
||||||
// The three genesis itself needs must be present, or the pivot cannot even begin — checked here
|
// The three genesis itself needs must be present, or the pivot cannot even begin — checked here
|
||||||
@@ -184,6 +206,8 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
|||||||
`--private-network wireguard`,
|
`--private-network wireguard`,
|
||||||
`--packet-filter nftables`,
|
`--packet-filter nftables`,
|
||||||
`--host ${HOST_PATH}`,
|
`--host ${HOST_PATH}`,
|
||||||
|
...(o.adopted ? [`--adopted`] : []),
|
||||||
|
...(o.flags ?? []),
|
||||||
// A service when the packaging was installed above (survives a reboot); otherwise the
|
// A service when the packaging was installed above (survives a reboot); otherwise the
|
||||||
// background process, which does not — the installer refuses to invent a unit either way.
|
// background process, which does not — the installer refuses to invent a unit either way.
|
||||||
...(o.hostService ? [] as string[] : [`--host-in-background`]),
|
...(o.hostService ? [] as string[] : [`--host-in-background`]),
|
||||||
@@ -193,20 +217,24 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
|||||||
// but because the installer is idempotent by design and says so, and because the one thing that
|
// but because the installer is idempotent by design and says so, and because the one thing that
|
||||||
// fails for a reason which goes away by itself is a pull: the store, broker and registry come
|
// fails for a reason which goes away by itself is a pull: the store, broker and registry come
|
||||||
// from the internet, and a rate-limited anonymous pull is not this mesh's fault.
|
// from the internet, and a rate-limited anonymous pull is not this mesh's fault.
|
||||||
let said = "";
|
|
||||||
let step = "";
|
let step = "";
|
||||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
const attempts = o.attempts ?? 3;
|
||||||
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
||||||
const ran = await on(command, 2_400_000);
|
const ran = await on(command, 2_400_000);
|
||||||
said = ran.out;
|
said = ran.out;
|
||||||
log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`);
|
log(`\n---- mesh-bootstrap on ${node} (attempt ${attempt}) ----\n${said}`);
|
||||||
if (ran.ok) { step = ""; break; }
|
if (ran.ok) { step = ""; break; }
|
||||||
step = stepIn(said);
|
step = stepIn(said) || "an unnamed step";
|
||||||
if (attempt < 3) {
|
if (attempt < attempts) {
|
||||||
log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`);
|
log(`genesis attempt ${attempt} stopped at ${step || "an unnamed step"}; re-running in 30s`);
|
||||||
await new Promise((r) => setTimeout(r, 30_000));
|
await new Promise((r) => setTimeout(r, 30_000));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));
|
if (step) return stop(step, said.split("\n").filter(Boolean).slice(-6).join("\n"));
|
||||||
|
if (o.verify === false) {
|
||||||
|
report.push(`\nThe installer finished; not asked whether it raised a mesh (verify: false).`);
|
||||||
|
return { ok: true, step: "", why: "", report, said };
|
||||||
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------------------------------------
|
// ------------------------------------------------------------------------------------------
|
||||||
// Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting
|
// Is it a WORKING MESH OF ONE? Asked of the machine, never inferred from the installer exiting
|
||||||
@@ -286,5 +314,5 @@ export async function genesis(o: GenesisOptions): Promise<GenesisResult> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`);
|
report.push(`\nVERDICT: ${node} is a working mesh of one, bootstrapped through the installer.`);
|
||||||
return { ok: true, step: "", why: "", report };
|
return { ok: true, step: "", why: "", report, said };
|
||||||
}
|
}
|
||||||
|
|||||||
+118
-7
@@ -40,6 +40,14 @@ const UPSTREAM_STORE =
|
|||||||
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
|
"postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee";
|
||||||
const UPSTREAM_BROKER =
|
const UPSTREAM_BROKER =
|
||||||
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
|
"cloudamqp/lavinmq@sha256:3eb54c12916d700a978c2ea86e6362cd4974b0e3189508718006d4e6d341246b";
|
||||||
|
/**
|
||||||
|
* And the bus, for `foundation-first-node-nats.lock` — the bundle the mesh raises since it stopped
|
||||||
|
* speaking AMQP (novox/hq ADR 0131). The digest is the bundle's own: what the registry served was a
|
||||||
|
* copy of the upstream image, so the same digest resolves on Docker Hub, and this is a prefix being
|
||||||
|
* removed rather than a reference being replaced.
|
||||||
|
*/
|
||||||
|
const UPSTREAM_BUS =
|
||||||
|
"nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The foundation bundle as a machine should receive it.
|
* The foundation bundle as a machine should receive it.
|
||||||
@@ -53,6 +61,7 @@ export function foundationBundle(path: string, held: HeldImage[]): string {
|
|||||||
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
|
text = text.replaceAll(/[A-Za-z0-9_.:-]+\/postgres@sha256:[0-9a-f]{64}/g, UPSTREAM_STORE);
|
||||||
text = text.replaceAll(
|
text = text.replaceAll(
|
||||||
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
|
/[A-Za-z0-9_.:-]+\/cloudamqp\/lavinmq@sha256:[0-9a-f]{64}/g, UPSTREAM_BROKER);
|
||||||
|
text = text.replaceAll(/[A-Za-z0-9_.:-]+:[0-9]+\/nats@sha256:[0-9a-f]{64}/g, UPSTREAM_BUS);
|
||||||
return pinnedInto(text, held);
|
return pinnedInto(text, held);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -237,14 +246,116 @@ export async function assertUniversalInvariants(
|
|||||||
/** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */
|
/** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */
|
||||||
export const FILTER_MODULE = "nftables";
|
export const FILTER_MODULE = "nftables";
|
||||||
|
|
||||||
/** The catalogue's manifest for a module, under either spelling of MESH_LAB_CATALOG: the modules
|
// --- the catalogue: a bed installs a module by reading its manifest, never by carrying a copy ----
|
||||||
* directory, or the checkout that holds it. */
|
|
||||||
export function catalogueManifest(module: string): string {
|
/**
|
||||||
const dir = process.env["MESH_LAB_CATALOG"] ?? "";
|
* The catalogue's `modules/` directory: MESH_LAB_CATALOG under either spelling (the checkout, or
|
||||||
for (const candidate of [resolve(dir, module, "module.json"), resolve(dir, "modules", module, "module.json")]) {
|
* its modules directory). Named, or absent — never guessed from a sibling path: the receipt claims
|
||||||
if (existsSync(candidate)) return candidate;
|
* the catalogue the run was pointed at (src/repos.ts), and a catalogue read from somewhere the
|
||||||
|
* receipt does not name is the drift this exists to close.
|
||||||
|
*
|
||||||
|
* Beds used to build the manifests they install inline, as literals copied from the catalogue when
|
||||||
|
* each bed was written. The copies did not move when the catalogue did, so a catalogue change was
|
||||||
|
* proven nowhere — and a bed that installs a copy proves the copy (novox/hq 04-ISSUES/073). A bed
|
||||||
|
* reads the catalogue, or it does not install a catalogue module; `beds-read-the-catalogue.test.ts`
|
||||||
|
* refuses an inline copy that names one.
|
||||||
|
*/
|
||||||
|
export function catalogueDir(): string {
|
||||||
|
const named = process.env["MESH_LAB_CATALOG"];
|
||||||
|
if (!named) throw new Error("MESH_LAB_CATALOG is not set to a checkout of mesh-catalog (or its modules directory)");
|
||||||
|
const candidates = [resolve(named, "modules"), resolve(named)];
|
||||||
|
for (const dir of candidates) {
|
||||||
|
// Known by the registry's manifest, which genesis reads from the catalogue and always will —
|
||||||
|
// not the control plane's, which lives in the control plane's own repository (ADR 0069).
|
||||||
|
if (existsSync(resolve(dir, "distribution", "module.json"))) return dir;
|
||||||
}
|
}
|
||||||
throw new Error(`no manifest for ${module} under MESH_LAB_CATALOG=${dir || "(unset)"}`);
|
throw new Error(`no catalogue: MESH_LAB_CATALOG=${named} and no distribution/module.json under ${candidates.join(" or ")}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a catalogue is where a bed will look — for a skip guard, which says so instead of failing. */
|
||||||
|
export function catalogueIsPresent(): string | false {
|
||||||
|
try { catalogueDir(); return false; } catch (err) { return (err as Error).message; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The catalogue's manifest for a module, as a path. */
|
||||||
|
export function catalogueManifest(module: string): string {
|
||||||
|
const path = resolve(catalogueDir(), module, "module.json");
|
||||||
|
if (!existsSync(path)) throw new Error(`no manifest for ${module} at ${path}`);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What the lab may rewrite in a catalogue manifest, and nothing else. */
|
||||||
|
export interface ForTheLab {
|
||||||
|
/**
|
||||||
|
* The image repository each build artifact was built as on this workstation, by artifact name.
|
||||||
|
* A module's own runtime is `mesh-runtime-<module>` by default — what `scripts/build-module-runtime.sh`
|
||||||
|
* tags and what the scenarios stock; a bed names it only where the scenario stocks another name.
|
||||||
|
* An artifact this does not name is refused: the bed must say what stands in for the builder.
|
||||||
|
*/
|
||||||
|
artifacts?: Record<string, string>;
|
||||||
|
/**
|
||||||
|
* Host-port remaps, where one machine carries modules whose published ports collide —
|
||||||
|
* `{ "8080": "8090:8080" }`, applied to every container of the module. The container side never
|
||||||
|
* changes.
|
||||||
|
*/
|
||||||
|
ports?: Record<string, string> | undefined;
|
||||||
|
/**
|
||||||
|
* Environment a container gets in the lab that it does not get in the mesh — an address the bed
|
||||||
|
* stands up in place of a real upstream, and nothing else. Merged over the manifest's own.
|
||||||
|
*/
|
||||||
|
env?: Record<string, Record<string, string>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A catalogue manifest as a machine in the lab can run it: the mesh's build section gone (the lab
|
||||||
|
* stocks images rather than building), each artifact replaced by the image the machine holds for it,
|
||||||
|
* every image pinned to what the machine holds or the upstream digest the catalogue pins, and the
|
||||||
|
* declared lab rewrites applied. Everything else is the catalogue's, verbatim — which is the point.
|
||||||
|
*/
|
||||||
|
export function catalogueModule(module: string, held: HeldImage[], lab: ForTheLab = {}): string {
|
||||||
|
const m = JSON.parse(readFileSync(catalogueManifest(module), "utf8")) as {
|
||||||
|
resources?: { id: string; type: string; image?: string; artifact?: string; ports?: string[]; env?: Record<string, string> }[];
|
||||||
|
build?: { artifacts?: { name: string; kind: string; from?: string }[] };
|
||||||
|
};
|
||||||
|
const artifacts: Record<string, string> = { runtime: `mesh-runtime-${module}`, ...(lab.artifacts ?? {}) };
|
||||||
|
// An upstream artifact is somebody else's image, which the mesh's builder copies into its own
|
||||||
|
// registry (ADR 0096). The lab stands in for the builder by using the reference the manifest
|
||||||
|
// pins, which the machine pulls over its uplink — the same bytes, without the copy.
|
||||||
|
const upstream = new Map<string, string>();
|
||||||
|
for (const a of m.build?.artifacts ?? []) {
|
||||||
|
if (a.kind === "upstream" && a.from) upstream.set(a.name, a.from);
|
||||||
|
}
|
||||||
|
for (const r of m.resources ?? []) {
|
||||||
|
if (r.type !== "container") continue;
|
||||||
|
if (typeof r.artifact === "string" && upstream.has(r.artifact) && !lab.artifacts?.[r.artifact]) {
|
||||||
|
r.image = onTheMachine(upstream.get(r.artifact)!, held);
|
||||||
|
delete r.artifact;
|
||||||
|
} else if (typeof r.artifact === "string") {
|
||||||
|
const repository = artifacts[r.artifact];
|
||||||
|
assert.ok(repository,
|
||||||
|
`${module}'s container '${r.id}' names the "${r.artifact}" artifact, which the mesh would ` +
|
||||||
|
`build. The lab does not build: the bed must say which stocked image stands in for it ` +
|
||||||
|
`(artifacts: { ${r.artifact}: "<repository>" }).`);
|
||||||
|
const reference = referenceFor(held, repository);
|
||||||
|
assert.ok(reference,
|
||||||
|
`${module}'s "${r.artifact}" artifact is ${repository} and this scenario stocked no such ` +
|
||||||
|
`image. Add it to the scenario's images: and build it (scripts/build-module-runtime.sh ${module}).`);
|
||||||
|
r.image = reference;
|
||||||
|
delete r.artifact;
|
||||||
|
} else if (typeof r.image === "string") {
|
||||||
|
r.image = onTheMachine(r.image, held);
|
||||||
|
}
|
||||||
|
if (lab.ports && Array.isArray(r.ports)) r.ports = r.ports.map((p) => lab.ports![p] ?? p);
|
||||||
|
const env = lab.env?.[r.id];
|
||||||
|
if (env) r.env = { ...(r.env ?? {}), ...env };
|
||||||
|
}
|
||||||
|
delete m.build;
|
||||||
|
return JSON.stringify(m);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a manifest's runtime dials the broker — the module then needs a scoped broker account. */
|
||||||
|
export function needsBrokerAccount(manifest: string): boolean {
|
||||||
|
return manifest.includes("MESH_BROKER_FILE");
|
||||||
}
|
}
|
||||||
|
|
||||||
function shellQuote(s: string): string {
|
function shellQuote(s: string): string {
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -39,7 +39,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "local-model-bed";
|
const SCENARIO = "local-model-bed";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -119,7 +119,8 @@ async function settled(withinMs = 600_000): Promise<void> {
|
|||||||
async function addAssign(name: string, manifest: string): Promise<void> {
|
async function addAssign(name: string, manifest: string): Promise<void> {
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
||||||
await mesh(`module add /${name}.json`);
|
await mesh(`module add /${name}.json`);
|
||||||
await mesh(`module issue ${name} --node ${MACHINE}`);
|
// No `module issue`: neither module speaks on the bus, and issuing a module with no broker
|
||||||
|
// secret to deliver into is refused (novox/hq issue 078).
|
||||||
await mesh(`assign ${MACHINE} ${name}`);
|
await mesh(`assign ${MACHINE} ${name}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,47 +154,13 @@ after(async () => {
|
|||||||
test("a node hosting a model answers model-access, and the consumer is handed its endpoint", {
|
test("a node hosting a model answers model-access, and the consumer is handed its endpoint", {
|
||||||
skip, timeout: 1_500_000,
|
skip, timeout: 1_500_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
const ollamaImage = pinned("ollama/ollama");
|
// Both manifests are the catalogue's own (novox/hq 04-ISSUES/073). The provider: ollama runs the
|
||||||
|
// model server and `provides: ["model-access"]` at node scope, serving its port and model. It mints
|
||||||
// The provider: ollama runs the model server and `provides: ["model-access"]` at node scope, serving
|
// nothing — provides/serves are declaration the mesh reads, so there is no runtime container, only
|
||||||
// its port and model. It mints nothing — provides/serves are declaration the mesh reads, so there is
|
// the server. The consumer requires model-access and is answered by the local node: no secret (the
|
||||||
// no runtime container, only the server.
|
// local server is keyless), only the bound endpoint facts, templated into an openai.env the mesh writes.
|
||||||
const ollamaManifest = JSON.stringify({
|
const ollamaManifest = catalogueModule("ollama", held);
|
||||||
module: "ollama",
|
const consumerManifest = catalogueModule("local-model-consumer", held);
|
||||||
version: "1",
|
|
||||||
capabilities: ["container-runtime"],
|
|
||||||
provides: [{ name: "model-access", scope: "node" }],
|
|
||||||
listens: [{ port: 11434, protocol: "tcp", from: "machine", why: "local consumers reaching the model server" }],
|
|
||||||
serves: { "model-access": { port: 11434, model: "llama3.2" } },
|
|
||||||
resources: [
|
|
||||||
{ id: "state", type: "directory", path: "/services/ollama", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "ollama",
|
|
||||||
image: ollamaImage, network: "host", env: { OLLAMA_HOST: "0.0.0.0:11434" },
|
|
||||||
volumes: ["/services/ollama:/root/.ollama"],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
// The consumer: it requires model-access and is answered by the local node. No secret (the local
|
|
||||||
// server is keyless), only the bound endpoint facts, templated into an openai.env the mesh writes.
|
|
||||||
const consumerManifest = JSON.stringify({
|
|
||||||
module: "local-model-consumer",
|
|
||||||
version: "1",
|
|
||||||
slug: "local",
|
|
||||||
requires: ["model-access"],
|
|
||||||
binds: { "model-access": "/var/lib/local-model-consumer/model.json" },
|
|
||||||
resources: [
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/local-model-consumer", mode: "0700" },
|
|
||||||
{ id: "config", type: "directory", path: "/var/lib/local-model-consumer/config", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "openai-env", type: "file", path: "/var/lib/local-model-consumer/config/openai.env", mode: "0600",
|
|
||||||
content:
|
|
||||||
"OPENAI_BASE_URL=http://${bound:model-access:at}:${bound:model-access:port}/v1\n" +
|
|
||||||
"OPENAI_MODEL=${bound:model-access:model}\nOPENAI_API_KEY=local\n",
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
await addAssign("ollama", ollamaManifest);
|
await addAssign("ollama", ollamaManifest);
|
||||||
await addAssign("local-model-consumer", consumerManifest);
|
await addAssign("local-model-consumer", consumerManifest);
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -36,7 +36,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "redis-node";
|
const SCENARIO = "redis-node";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -139,51 +139,12 @@ after(async () => {
|
|||||||
test("the mesh grants a consumer redis's cache, and the credential it delivers authenticates", {
|
test("the mesh grants a consumer redis's cache, and the credential it delivers authenticates", {
|
||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
// The PROVIDER: redis in its committed shape — server and a broker-bound runtime on the private
|
// The PROVIDER: the catalogue's redis (novox/hq 04-ISSUES/074) — server and a broker-bound
|
||||||
// redis network, the runtime running the provisioner.
|
// runtime on the private redis network, the runtime running the provisioner. It requires a
|
||||||
const redisManifest = JSON.stringify({
|
// `secret` for its own password, so the vault that provides one is installed beside it, exactly
|
||||||
module: "redis",
|
// as the vault bed does.
|
||||||
version: "1",
|
const vaultManifest = catalogueModule("mesh-vault", held);
|
||||||
provides: [{ name: "redis-cache", scope: "mesh" }],
|
const redisManifest = catalogueModule("redis", held);
|
||||||
serves: { "redis-cache": {} },
|
|
||||||
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
|
|
||||||
grants: { "redis-cache": "/var/lib/redis-module/grants" },
|
|
||||||
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
|
|
||||||
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
|
|
||||||
{
|
|
||||||
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
|
|
||||||
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
|
|
||||||
},
|
|
||||||
{ id: "net", type: "network", name: "redis" },
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "redis",
|
|
||||||
ports: ["6379"],
|
|
||||||
volumes: ["/services/redis/data:/data", "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro"],
|
|
||||||
args: ["/etc/redis/redis.conf"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
|
|
||||||
network: "redis",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro",
|
|
||||||
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
|
|
||||||
MESH_PROVISION_REDIS: "redis:6379",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
// The CONSUMER: a module that requires redis-cache and no more. It runs no code here — the mesh
|
// The CONSUMER: a module that requires redis-cache and no more. It runs no code here — the mesh
|
||||||
// delivers it a bound file (where redis is, and the login to present) and its sealed password,
|
// delivers it a bound file (where redis is, and the login to present) and its sealed password,
|
||||||
@@ -191,6 +152,9 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
|
|||||||
const consumerManifest = JSON.stringify({
|
const consumerManifest = JSON.stringify({
|
||||||
module: "cacheuser",
|
module: "cacheuser",
|
||||||
version: "1",
|
version: "1",
|
||||||
|
// `mesh_anchor_cacheuser` is 21 characters, over the 20 a backend keeps (ADR 0049); the slug
|
||||||
|
// makes the consumer identity `mesh_anchor_cache`.
|
||||||
|
slug: "cache",
|
||||||
requires: ["redis-cache"],
|
requires: ["redis-cache"],
|
||||||
// `contributes` (not just `requires`) is what makes a consumer *ask* — the grant forms from a
|
// `contributes` (not just `requires`) is what makes a consumer *ask* — the grant forms from a
|
||||||
// contribution. It must be non-empty; redis's provisioner ignores the value (it uses the login
|
// contribution. It must be non-empty; redis's provisioner ignores the value (it uses the login
|
||||||
@@ -201,6 +165,10 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
|
|||||||
resources: [{ id: "state", type: "directory", path: "/var/lib/cacheuser", mode: "0700" }],
|
resources: [{ id: "state", type: "directory", path: "/var/lib/cacheuser", mode: "0700" }],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
await must(`printf %s ${quote(vaultManifest)} > /tmp/mesh-vault.json && docker cp /tmp/mesh-vault.json mesh-controller:/mesh-vault.json`);
|
||||||
|
await mesh("module add /mesh-vault.json");
|
||||||
|
await mesh(`module issue mesh-vault --node ${MACHINE}`);
|
||||||
|
await mesh(`assign ${MACHINE} mesh-vault`);
|
||||||
await must(`printf %s ${quote(redisManifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
|
await must(`printf %s ${quote(redisManifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
|
||||||
await mesh("module add /redis.json");
|
await mesh("module add /redis.json");
|
||||||
await mesh(`module issue redis --node ${MACHINE}`);
|
await mesh(`module issue redis --node ${MACHINE}`);
|
||||||
@@ -252,4 +220,24 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
|
|||||||
assert.doesNotMatch(authed.out, /WRONGPASS|NOPERM|no password/i,
|
assert.doesNotMatch(authed.out, /WRONGPASS|NOPERM|no password/i,
|
||||||
`the consumer's mesh-delivered credential did not authenticate — the two ends do not agree:\n${authed.out}`);
|
`the consumer's mesh-delivered credential did not authenticate — the two ends do not agree:\n${authed.out}`);
|
||||||
assert.match(authed.out, /PONG/, `expected PONG authenticating as the granted consumer:\n${authed.out}`);
|
assert.match(authed.out, /PONG/, `expected PONG authenticating as the granted consumer:\n${authed.out}`);
|
||||||
|
|
||||||
|
// And the provider needed no seal key to do it: the password reached it as a file the host left
|
||||||
|
// after unsealing, so MESH_SEAL_KEY is set nowhere (novox/hq ADR 0048). Carried over from the
|
||||||
|
// retired provider-uses-mesh-credential bed, whose other proofs this bed makes with the mesh
|
||||||
|
// writing the contributions rather than the bed.
|
||||||
|
const runtimeEnv = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`);
|
||||||
|
assert.doesNotMatch(runtimeEnv, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${runtimeEnv}`);
|
||||||
|
|
||||||
|
// A grant means exactly the consumer's own keys — `<login>:*`, the keyspace redis's provisioner
|
||||||
|
// scopes the ACL user to: under it the consumer reads and writes, outside it and on the server as
|
||||||
|
// a whole it is refused. Carried over from the large mesh bed's retired cache-grant test —
|
||||||
|
// without this a provisioner that granted everything would keep every bed green.
|
||||||
|
const asConsumer = (command: string) =>
|
||||||
|
on(`docker exec redis redis-cli --user ${quote(as)} --pass ${quote(password)} --no-auth-warning ${command} 2>&1`);
|
||||||
|
assert.match((await asConsumer(`SET ${as}:proof yes`)).out, /OK/, "the consumer cannot write under its own login");
|
||||||
|
assert.match((await asConsumer(`GET ${as}:proof`)).out, /yes/, "the consumer cannot read back what it wrote");
|
||||||
|
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
|
||||||
|
"the consumer wrote outside its own keys, so the grant means more than it says");
|
||||||
|
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
|
||||||
|
"the consumer flushed the whole server, so the grant means more than it says");
|
||||||
});
|
});
|
||||||
|
|||||||
+107
-531
@@ -18,13 +18,13 @@
|
|||||||
|
|
||||||
import { test, before, after } from "node:test";
|
import { test, before, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts";
|
||||||
import { incus } from "../../src/incus/client.ts";
|
import { incus } from "../../src/incus/client.ts";
|
||||||
import { machineName } from "../../src/lifecycle/names.ts";
|
import { machineName } from "../../src/lifecycle/names.ts";
|
||||||
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
|
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
|
||||||
@@ -37,7 +37,6 @@ const binary = hostBinaryPath();
|
|||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
|
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
|
||||||
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */
|
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */
|
||||||
const moduleExamples = process.env["MESH_LAB_MODULES"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
const skip = !capability.usable
|
||||||
? `lab not usable: ${capability.why}`
|
? `lab not usable: ${capability.why}`
|
||||||
@@ -45,7 +44,8 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
// The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074).
|
||||||
|
: catalogueIsPresent() || false;
|
||||||
|
|
||||||
const SCENARIO = "two-nodes";
|
const SCENARIO = "two-nodes";
|
||||||
let instanceId = "";
|
let instanceId = "";
|
||||||
@@ -200,6 +200,24 @@ function tokenFrom(said: string): string {
|
|||||||
return found;
|
return found;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The builder started by hand on the anchor, against the foundation broker's plain port on
|
||||||
|
* loopback — the one that builds until a builder module can (see the retired test's note). */
|
||||||
|
async function startBuilder(): Promise<void> {
|
||||||
|
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
|
||||||
|
// return, so it is pushed whenever the machine has none.
|
||||||
|
if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) {
|
||||||
|
await incus([
|
||||||
|
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
|
||||||
|
"--mode", "0755",
|
||||||
|
], 180_000);
|
||||||
|
}
|
||||||
|
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
||||||
|
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
|
||||||
|
`(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
||||||
|
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
||||||
|
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
|
||||||
|
}
|
||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
if (skip) return;
|
if (skip) return;
|
||||||
|
|
||||||
@@ -232,6 +250,8 @@ before(async () => {
|
|||||||
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
|
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
|
||||||
assert.equal(running.out.trim(), "yes",
|
assert.equal(running.out.trim(), "yes",
|
||||||
"the host did not come back after a restore, so nothing would apply anything");
|
"the host did not come back after a restore, so nothing would apply anything");
|
||||||
|
// The hand-started builder is memory too, and the snapshot is disk.
|
||||||
|
if (builder) await startBuilder();
|
||||||
|
|
||||||
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
|
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
|
||||||
`and started the host again`);
|
`and started the host again`);
|
||||||
@@ -258,17 +278,7 @@ before(async () => {
|
|||||||
|
|
||||||
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
||||||
// assumed: nothing else in this scenario would start one.
|
// assumed: nothing else in this scenario would start one.
|
||||||
if (builder) {
|
if (builder) await startBuilder();
|
||||||
await incus([
|
|
||||||
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
|
|
||||||
"--mode", "0755",
|
|
||||||
], 180_000);
|
|
||||||
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
|
||||||
await must("anchor",
|
|
||||||
`MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
|
||||||
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
|
||||||
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
|
|
||||||
}
|
|
||||||
if (warming) {
|
if (warming) {
|
||||||
// Snapshotted only now, with everything up: a state worth returning to is the one after the
|
// Snapshotted only now, with everything up: a state worth returning to is the one after the
|
||||||
// part nobody wants to repeat.
|
// part nobody wants to repeat.
|
||||||
@@ -318,7 +328,7 @@ test("both machines join it, and the token is all they need", { skip, timeout: 9
|
|||||||
test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
|
test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
|
||||||
// The whole argument, on real machines: the two ends must hold the SAME password, and it must
|
// The whole argument, on real machines: the two ends must hold the SAME password, and it must
|
||||||
// appear nowhere the mesh or the broker could read it.
|
// appear nowhere the mesh or the broker could read it.
|
||||||
await must("anchor", `printf %s '{"module":"postgres","version":"1",` +
|
await must("anchor", `printf %s '{"module":"a-store","version":"1",` +
|
||||||
`"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` +
|
`"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` +
|
||||||
`"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` +
|
`"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` +
|
||||||
`"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
|
`"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
|
||||||
@@ -326,7 +336,8 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
|
|||||||
// real program takes a credential: a sealed file is a password alone, and almost nothing reads
|
// real program takes a credential: a sealed file is a password alone, and almost nothing reads
|
||||||
// one. The mesh cannot compose the document — it discarded the value — so the module supplies it
|
// one. The mesh cannot compose the document — it discarded the value — so the module supplies it
|
||||||
// with `${secret:...}` in it and the host, the only thing that sees both halves, fills it in.
|
// with `${secret:...}` in it and the host, the only thing that sees both halves, fills it in.
|
||||||
await must("anchor", `printf %s '{"module":"meshboard","version":"1",` +
|
// A slug, so its identity on a backend stays within an S3 access key's 20 characters (ADR 0049).
|
||||||
|
await must("anchor", `printf %s '{"module":"meshboard","version":"1","slug":"board",` +
|
||||||
`"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` +
|
`"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` +
|
||||||
`"binds":{"postgres-database":"/etc/meshboard/database.json"},` +
|
`"binds":{"postgres-database":"/etc/meshboard/database.json"},` +
|
||||||
`"secrets":{"postgres-database":"/etc/meshboard/database.password"},` +
|
`"secrets":{"postgres-database":"/etc/meshboard/database.password"},` +
|
||||||
@@ -342,14 +353,20 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
|
|||||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||||
await mesh("overlay place laptop --site lab");
|
await mesh("overlay place laptop --site lab");
|
||||||
for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
|
for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
|
||||||
await mesh("assign anchor postgres");
|
await mesh("assign anchor a-store");
|
||||||
await mesh("assign laptop meshboard");
|
await mesh("assign laptop meshboard");
|
||||||
|
|
||||||
for (const machine of ["anchor", "laptop"]) {
|
for (const machine of ["anchor", "laptop"]) {
|
||||||
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
// Once. On a warm return the host is already running (see `before`); a second one would
|
||||||
|
// consume the same queue and apply the same declaration twice, concurrently.
|
||||||
|
await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
||||||
}
|
}
|
||||||
await mesh("push");
|
await mesh("push");
|
||||||
await new Promise((r) => setTimeout(r, 8000));
|
await new Promise((r) => setTimeout(r, 8000));
|
||||||
|
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
||||||
|
// and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
|
||||||
|
// keeps the hub closed and nothing on the laptop reaches anchor over the private network.
|
||||||
|
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
||||||
|
|
||||||
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
|
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
|
||||||
// Named after the machine *and* the module, because a consumer is both (novox/hq
|
// Named after the machine *and* the module, because a consumer is both (novox/hq
|
||||||
@@ -371,7 +388,7 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
|
|||||||
assert.match(filled, /^PGPASSWORD=.+$/m, `the password was never put in:\n${filled}`);
|
assert.match(filled, /^PGPASSWORD=.+$/m, `the password was never put in:\n${filled}`);
|
||||||
assert.ok(filled.includes(`PGPASSWORD=${onConsumer}`),
|
assert.ok(filled.includes(`PGPASSWORD=${onConsumer}`),
|
||||||
`the file holds a different password from the credential file:\n${filled}`);
|
`the file holds a different password from the credential file:\n${filled}`);
|
||||||
assert.match(filled, /^PGUSER=mesh_laptop_meshboard$/m,
|
assert.match(filled, /^PGUSER=mesh_laptop_board$/m,
|
||||||
`the consumer was not told what name to present:\n${filled}`);
|
`the consumer was not told what name to present:\n${filled}`);
|
||||||
assert.match(filled, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${filled}`);
|
assert.match(filled, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${filled}`);
|
||||||
assert.doesNotMatch(filled, /\$\{/,
|
assert.doesNotMatch(filled, /\$\{/,
|
||||||
@@ -617,6 +634,7 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
|
|||||||
`"capabilities":["container-runtime"],` +
|
`"capabilities":["container-runtime"],` +
|
||||||
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
|
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
|
||||||
`"serves":{"artifact-store":{"port":5000}},` +
|
`"serves":{"artifact-store":{"port":5000}},` +
|
||||||
|
`"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` +
|
||||||
`"resources":[` +
|
`"resources":[` +
|
||||||
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
|
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
|
||||||
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
|
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
|
||||||
@@ -630,10 +648,15 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
|
|||||||
await mesh("module add /registry.json");
|
await mesh("module add /registry.json");
|
||||||
await mesh("assign anchor registry");
|
await mesh("assign anchor registry");
|
||||||
await mesh("push anchor");
|
await mesh("push anchor");
|
||||||
await new Promise((r) => setTimeout(r, 12_000));
|
// The store's image is pulled from upstream at apply, over the uplink; that takes what it takes.
|
||||||
|
let names = "";
|
||||||
|
for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) {
|
||||||
|
await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
names = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||||
|
}
|
||||||
// Running, and answering — a container that is up is not a registry that replies.
|
// Running, and answering — a container that is up is not a registry that replies.
|
||||||
assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
|
assert.match(names, /mesh-registry/,
|
||||||
|
`the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`);
|
||||||
let answers = false;
|
let answers = false;
|
||||||
for (let i = 0; i < 20 && !answers; i++) {
|
for (let i = 0; i < 20 && !answers; i++) {
|
||||||
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
|
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
|
||||||
@@ -654,8 +677,11 @@ test("a machine serves its internal name with a certificate the mesh issued", {
|
|||||||
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
|
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
|
||||||
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
|
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
|
||||||
// moment something connects, which is the worst place to find out.
|
// moment something connects, which is the worst place to find out.
|
||||||
|
// The port the handshake below is tried on, declared: the anchor filters what its modules
|
||||||
|
// did not declare (ADR 0088), and a test server on an undeclared port proves only that.
|
||||||
await must("anchor", `printf %s '{"module":"served","version":"1",` +
|
await must("anchor", `printf %s '{"module":"served","version":"1",` +
|
||||||
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
|
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
|
||||||
|
`"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` +
|
||||||
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
|
||||||
`> /tmp/served.json`);
|
`> /tmp/served.json`);
|
||||||
await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
|
await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
|
||||||
@@ -814,117 +840,11 @@ test("a machine filters exactly what its modules declared, and nothing else", {
|
|||||||
"the port stayed open after the module that wanted it was removed");
|
"the port stayed open after the module that wanted it was removed");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the builder is a module the mesh assigns, with a credential the mesh delivered", {
|
// The builder as a module the mesh assigns, with a credential the mesh delivered, is what genesis
|
||||||
skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
|
// proves now (genesis-single installs the catalogue's builder through the installer, novox/hq ADR
|
||||||
timeout: 900_000,
|
// 0069). The test that lived here declared the builder's image as an upstream artifact by the bare
|
||||||
}, async () => {
|
// image ID the lab holds, which is not a reference a registry copy can fetch (ADR 0096); retired
|
||||||
// Until this, the builder was a program somebody started on a machine with whatever credential
|
// 2026-09-21 rather than rewritten into a second genesis.
|
||||||
// they had to hand — in practice the broker's administrative one. A program documented as
|
|
||||||
// holding its own credential and given somebody else's is worse than one with no story at all.
|
|
||||||
//
|
|
||||||
// So: the mesh issues a scoped account, seals it to the machine, and delivers it with the
|
|
||||||
// declaration. Nobody types it and the mesh cannot read it back.
|
|
||||||
await must("anchor", `mkdir -p /root/builder && printf %s '{"module":"builder","version":"1",` +
|
|
||||||
`"requires":["artifact-store"],"capabilities":["container-runtime"],` +
|
|
||||||
`"claims":[{"name":"the-build-machine","scope":"node"}],` +
|
|
||||||
`"binds":{"artifact-store":"/var/lib/mesh/builder/artifact-store.json"},` +
|
|
||||||
`"own-secrets":{"broker":"/var/lib/mesh/builder/broker"},` +
|
|
||||||
`"build":{"artifacts":[{"name":"builder","kind":"upstream",` +
|
|
||||||
`"from":"${pinned("mesh-builder")}"}]},` +
|
|
||||||
`"resources":[` +
|
|
||||||
`{"id":"state","type":"directory","path":"/var/lib/mesh/builder","mode":"0700"},` +
|
|
||||||
`{"id":"workspace","type":"directory","path":"/var/lib/mesh/builder/workspace","mode":"0700"},` +
|
|
||||||
`{"id":"run","type":"container","name":"mesh-builder","artifact":"builder",` +
|
|
||||||
`"network":"host",` +
|
|
||||||
`"volumes":["/var/lib/mesh/builder:/var/lib/mesh/builder",` +
|
|
||||||
`"/var/run/docker.sock:/var/run/docker.sock"],` +
|
|
||||||
`"env":{"MESH_BROKER_FILE":"/var/lib/mesh/builder/broker",` +
|
|
||||||
`"MESH_BINDING":"/var/lib/mesh/builder/artifact-store.json",` +
|
|
||||||
`"MESH_WORKSPACE":"/var/lib/mesh/builder/workspace"}}]}' > /root/builder/module.json`);
|
|
||||||
await must("anchor", `cd /root/builder && git init -q . && git add -A && ` +
|
|
||||||
`git -c user.email=lab -c user.name=lab commit -qm builder`);
|
|
||||||
|
|
||||||
// The builder's own image is built by the builder that is already running — the same
|
|
||||||
// chicken-and-egg as the registry, resolved the same way. The one started by hand does this
|
|
||||||
// last piece of work and is then replaced by the module it just built.
|
|
||||||
await mesh("build /root/builder --wait 300s", 420_000);
|
|
||||||
|
|
||||||
// The mesh makes the account and seals the URL to this machine. Nothing is printed that would
|
|
||||||
// work if it were pasted somewhere else.
|
|
||||||
const issued = await mesh("builder issue lab-builder --node anchor");
|
|
||||||
assert.match(issued, /sealed to anchor/, issued);
|
|
||||||
assert.doesNotMatch(issued, /amqps:\/\/lab-builder:/,
|
|
||||||
"the credential was printed, so the one copy that matters is on a terminal");
|
|
||||||
|
|
||||||
// Now the hand-started one goes, or two builders race for the same queue and whichever answers
|
|
||||||
// proves nothing. By process name: `pkill -f` matches the shell running it too, which kills the
|
|
||||||
// connection carrying the command and hangs the caller waiting for a reply that will never
|
|
||||||
// come. Cost an hour once, in this file.
|
|
||||||
await on("anchor", `pkill -x mesh-builder`);
|
|
||||||
await new Promise((r) => setTimeout(r, 2000));
|
|
||||||
assert.ok(!(await on("anchor", `pgrep -x mesh-builder`)).ok,
|
|
||||||
"the hand-started builder is still running, so this would test that one");
|
|
||||||
|
|
||||||
await mesh("assign anchor builder");
|
|
||||||
await mesh("push anchor");
|
|
||||||
await new Promise((r) => setTimeout(r, 20_000));
|
|
||||||
|
|
||||||
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
||||||
assert.match(running, /mesh-builder/,
|
|
||||||
`the builder was assigned and is not running:\n${running}\n` +
|
|
||||||
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
|
||||||
|
|
||||||
// Running is not connected. A builder that cannot reach the broker sits there, and every
|
|
||||||
// outward sign — the container is up, the credential is on disk — says it is working.
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
const said = await on("anchor", `docker logs mesh-builder 2>&1 | tail -20`);
|
|
||||||
assert.doesNotMatch(said.out, /cannot reach the broker/,
|
|
||||||
`the builder is running and cannot reach the broker:\n${said.out}`);
|
|
||||||
|
|
||||||
// The credential arrived, is readable only by the machine, and is the scoped account rather
|
|
||||||
// than the broker's own.
|
|
||||||
assert.match(await must("anchor", `stat -c %a /var/lib/mesh/builder/broker`), /^600/);
|
|
||||||
const credential = await must("anchor", `cat /var/lib/mesh/builder/broker`);
|
|
||||||
assert.match(credential, /"url":"amqps:\/\/lab-builder:/,
|
|
||||||
"the builder is using an account that is not its own");
|
|
||||||
assert.doesNotMatch(credential, /guest:guest/, "the builder holds the broker's own account");
|
|
||||||
// And what to check the broker against. A mesh's broker presents a certificate of the mesh's
|
|
||||||
// own, so a URL alone reaches only a broker some public authority vouches for — which is no
|
|
||||||
// mesh broker at all, and fails at TLS with an error about an unknown authority.
|
|
||||||
assert.match(credential, /"fingerprint":"(sha256:)?[0-9a-f]{64}"/,
|
|
||||||
`the builder was given nothing to verify the broker with:\n${credential}`);
|
|
||||||
|
|
||||||
// And it works: the mesh asks this builder to build something, and it does. Answering is the
|
|
||||||
// only proof that the delivered credential authenticates — a container that is up with a
|
|
||||||
// credential it cannot use looks identical from outside.
|
|
||||||
// Somewhere the builder can actually see. A builder that is a module runs in a container, so
|
|
||||||
// the machine's filesystem is not its own — a path like /root only works for a builder somebody
|
|
||||||
// started on the host, which is what the first build above used. In a real mesh a module is
|
|
||||||
// cloned from the forge over a URL; here it goes in the directory the module already mounts,
|
|
||||||
// which is the same fact wearing different clothes.
|
|
||||||
const repo = "/var/lib/mesh/builder/repositories/built";
|
|
||||||
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"built","version":"1",` +
|
|
||||||
`"resources":[{"id":"marker","type":"file","path":"/etc/built","content":"yes","mode":"0644"}]}' ` +
|
|
||||||
`> ${repo}/module.json`);
|
|
||||||
await must("anchor", `cd ${repo} && git init -q . && git add -A && ` +
|
|
||||||
`git -c user.email=lab -c user.name=lab commit -qm built`);
|
|
||||||
try {
|
|
||||||
await mesh(`build ${repo} --wait 300s`, 420_000);
|
|
||||||
} catch (why) {
|
|
||||||
// The builder's own account of itself. Without it the failure is "nothing consumed the
|
|
||||||
// queue", which names no cause and is the same sentence whether the credential was refused,
|
|
||||||
// the queue was never declared, or the process died three seconds in.
|
|
||||||
const said = (await on("anchor", `docker logs mesh-builder 2>&1 | tail -40`)).out;
|
|
||||||
throw new Error(`${(why as Error).message}\n\nwhat the builder said:\n${said}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Naming the module, and not merely containing its name: `builds` says "nothing has been built
|
|
||||||
// yet" when there is nothing, and that sentence contains the word this was matching on.
|
|
||||||
const recorded = await mesh("builds built");
|
|
||||||
assert.doesNotMatch(recorded, /nothing has been built/,
|
|
||||||
`the build was accepted and no build was recorded against the module:\n${recorded}`);
|
|
||||||
assert.match(recorded, /built/, recorded);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("rotating a credential moves both ends, and the old one stops working", {
|
test("rotating a credential moves both ends, and the old one stops working", {
|
||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
@@ -1072,7 +992,7 @@ test("a route is a grant: a workload is reached by the name it asked for", {
|
|||||||
// they are different questions: one says who may reach it, the other says by what name — and
|
// they are different questions: one says who may reach it, the other says by what name — and
|
||||||
// the earlier test left this machine filtering, so a module that asked for a route and not for
|
// the earlier test left this machine filtering, so a module that asked for a route and not for
|
||||||
// the port would be unreachable by the proxy it just asked for.
|
// the port would be unreachable by the proxy it just asked for.
|
||||||
await must("anchor", `printf %s '{"module":"storefront","version":"1",` +
|
await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` +
|
||||||
`"requires":["route"],"capabilities":["container-runtime"],` +
|
`"requires":["route"],"capabilities":["container-runtime"],` +
|
||||||
`"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
|
`"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
|
||||||
`"binds":{"route":"/etc/storefront/route.json"},` +
|
`"binds":{"route":"/etc/storefront/route.json"},` +
|
||||||
@@ -1236,6 +1156,10 @@ test("a new commit reaches a machine that is already running the old one", {
|
|||||||
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
|
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
|
||||||
// question "did my change go out?". This is that question, end to end — a commit, a build, a
|
// question "did my change go out?". This is that question, end to end — a commit, a build, a
|
||||||
// catalogue, and a machine that ends up running what the source says.
|
// catalogue, and a machine that ends up running what the source says.
|
||||||
|
// The hand-started builder does not outlive a broker restart, and the foundation's broker is
|
||||||
|
// recreated when the first push reconciles it: a builder is (re)started here, where a build is
|
||||||
|
// asked for. The mesh's own builder is a module with a restart policy and needs none of this.
|
||||||
|
await startBuilder();
|
||||||
const repo = "/var/lib/mesh/builder/repositories/delivered";
|
const repo = "/var/lib/mesh/builder/repositories/delivered";
|
||||||
const write = async (what: string) =>
|
const write = async (what: string) =>
|
||||||
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
|
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
|
||||||
@@ -1363,79 +1287,11 @@ test("the board names the machine that is not doing what it was told", {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
|
// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
|
||||||
test("the hub can be filtered without severing the mesh", {
|
// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on
|
||||||
skip, timeout: 900_000,
|
// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a
|
||||||
}, async () => {
|
// filter module derives the rules — so the credential test above assigns the catalogue's and
|
||||||
// The machine that most needs a firewall was the one that could not have one. A hub is dialled
|
// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh
|
||||||
// by every node at other sites; a machine that is not a hub dials out and needs nothing open.
|
// was not severed. Retired 2026-09-22.
|
||||||
// They are the same module, so a static `listens` cannot say it — and the machine it gets wrong
|
|
||||||
// is the one facing the public internet.
|
|
||||||
//
|
|
||||||
// The failure this guards against is not subtle and is very hard to recover from: a rule set
|
|
||||||
// that closes the hub's own port takes the private network down, and the mesh's way of fixing
|
|
||||||
// anything is to send a declaration over it.
|
|
||||||
// Its own directory. Another module on this machine already declares /etc/mesh, and the mesh
|
|
||||||
// refuses two modules declaring one path rather than letting the second quietly win — which it
|
|
||||||
// did here, correctly, the first time this ran.
|
|
||||||
const rules = "/etc/mesh-hub/filter.nft";
|
|
||||||
await must("anchor", `printf %s '{"module":"hubfilter","version":"1",` +
|
|
||||||
`"capabilities":["firewall"],` +
|
|
||||||
`"filtering":{"into":"${rules}"},` +
|
|
||||||
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
|
|
||||||
`{"id":"dir","type":"directory","path":"/etc/mesh-hub","mode":"0755"},` +
|
|
||||||
`{"id":"unit","type":"file","path":"/etc/systemd/system/hub-filter.service",` +
|
|
||||||
`"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for the hub\\n` +
|
|
||||||
`[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
|
|
||||||
`ExecStart=/usr/bin/nft -f ${rules}\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
|
|
||||||
`{"id":"filter","type":"service","unit":"hub-filter.service","state":"running",` +
|
|
||||||
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/hubfilter.json`);
|
|
||||||
await must("anchor", `docker cp /tmp/hubfilter.json mesh-controller:/hubfilter.json`);
|
|
||||||
await mesh("module add /hubfilter.json");
|
|
||||||
await mesh("assign anchor hubfilter");
|
|
||||||
await mesh("push anchor");
|
|
||||||
await new Promise((r) => setTimeout(r, 20_000));
|
|
||||||
|
|
||||||
// The hub's own way onto the private network is open, and derived — nothing in that manifest
|
|
||||||
// mentions a port.
|
|
||||||
const written = await must("anchor", `cat ${rules}`);
|
|
||||||
assert.match(written, /udp dport 51820 accept/,
|
|
||||||
`the hub's rule set closes the private network it is the way onto:\n${written}`);
|
|
||||||
// The module that provides the private network, not the requirement it answers: `networking`
|
|
||||||
// is the domain a module offers, and what caused a rule is the module itself.
|
|
||||||
assert.match(written, /# mesh-wireguard — the private network/,
|
|
||||||
`the rule does not name what caused it:\n${written}`);
|
|
||||||
|
|
||||||
// Loaded, and the mesh still works: a declaration reaches the other machine, which it cannot if
|
|
||||||
// the overlay is severed. This is the assertion that matters — a rule file that looks right and
|
|
||||||
// a mesh that has stopped are exactly what this is guarding against.
|
|
||||||
assert.match(await must("anchor", `nft list table inet mesh`), /dport 51820/);
|
|
||||||
|
|
||||||
await must("laptop", `rm -f /etc/mesh-still-works`);
|
|
||||||
await must("anchor", `printf %s '{"module":"stillworks","version":"1",` +
|
|
||||||
`"resources":[{"id":"marker","type":"file","path":"/etc/mesh-still-works",` +
|
|
||||||
`"content":"yes","mode":"0644"}]}' > /tmp/stillworks.json`);
|
|
||||||
await must("anchor", `docker cp /tmp/stillworks.json mesh-controller:/stillworks.json`);
|
|
||||||
await mesh("module add /stillworks.json");
|
|
||||||
await mesh("assign laptop stillworks");
|
|
||||||
await mesh("push laptop");
|
|
||||||
|
|
||||||
let arrived = false;
|
|
||||||
for (let i = 0; i < 20 && !arrived; i++) {
|
|
||||||
arrived = (await on("laptop", `test -f /etc/mesh-still-works`)).ok;
|
|
||||||
if (!arrived) await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.ok(arrived,
|
|
||||||
"the hub applied its own rule set and the mesh stopped reaching the other machine");
|
|
||||||
|
|
||||||
// And the other machine still reaches the hub over the private network, which is what the
|
|
||||||
// opened port is for.
|
|
||||||
assert.ok((await on("laptop", `ping -c 1 -W 5 anchor.internal`)).ok,
|
|
||||||
"the private network is down after the hub filtered itself");
|
|
||||||
|
|
||||||
await mesh("unassign anchor hubfilter");
|
|
||||||
await mesh("unassign laptop stillworks");
|
|
||||||
await mesh("push");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a container reaches another machine by the name the mesh gave it", {
|
test("a container reaches another machine by the name the mesh gave it", {
|
||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
@@ -1480,6 +1336,23 @@ test("a container reaches another machine by the name the mesh gave it", {
|
|||||||
|
|
||||||
// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
|
// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
|
||||||
// told each one.
|
// told each one.
|
||||||
|
/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so
|
||||||
|
* it is issued once per machine. The mesh writes the resolver's data as a fact the module
|
||||||
|
* declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */
|
||||||
|
const resolverIssued = new Set<string>();
|
||||||
|
async function resolverModules(machines: string[]): Promise<void> {
|
||||||
|
for (const name of ["dnsmasq", "resolved-split-dns"]) {
|
||||||
|
const manifest = catalogueModule(name, held);
|
||||||
|
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
||||||
|
await mesh(`module add /${name}.json`);
|
||||||
|
}
|
||||||
|
for (const machine of machines) {
|
||||||
|
if (resolverIssued.has(machine)) continue;
|
||||||
|
await mesh(`module issue dnsmasq --node ${machine}`);
|
||||||
|
resolverIssued.add(machine);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
test("every name under a machine resolves to that machine", {
|
test("every name under a machine resolves to that machine", {
|
||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
@@ -1490,9 +1363,11 @@ test("every name under a machine resolves to that machine", {
|
|||||||
//
|
//
|
||||||
// The mesh writes the data and runs no daemon: a resolver is third-party software, and the
|
// The mesh writes the data and runs no daemon: a resolver is third-party software, and the
|
||||||
// mesh has no business choosing one. So what is checked here is the mesh's half — that the
|
// mesh has no business choosing one. So what is checked here is the mesh's half — that the
|
||||||
// data is right, complete, and follows the machines.
|
// data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq
|
||||||
await mesh("assign anchor mesh-resolver");
|
// declares, so that module is what is assigned; what it runs is the next test's concern.
|
||||||
await mesh("assign laptop mesh-resolver");
|
await resolverModules(["anchor", "laptop"]);
|
||||||
|
await mesh("assign anchor dnsmasq");
|
||||||
|
await mesh("assign laptop dnsmasq");
|
||||||
await mesh("push");
|
await mesh("push");
|
||||||
await new Promise((r) => setTimeout(r, 15_000));
|
await new Promise((r) => setTimeout(r, 15_000));
|
||||||
|
|
||||||
@@ -1518,10 +1393,10 @@ test("every name under a machine resolves to that machine", {
|
|||||||
// because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
|
// because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
|
||||||
// fails at once and says which name it was.
|
// fails at once and says which name it was.
|
||||||
//
|
//
|
||||||
// Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the
|
// Both: the resolver's data follows the private network, so the machine leaves the network as
|
||||||
// network, so unassigning the domain module alone leaves the machine on the network — pulled
|
// well as the resolver, and what is asserted is that the machine that stayed is answered for and
|
||||||
// back by its own requirement. The mesh was right and this test was wrong the first time.
|
// the one that left is not.
|
||||||
await mesh("unassign laptop mesh-resolver");
|
await mesh("unassign laptop dnsmasq");
|
||||||
await mesh("unassign laptop networking");
|
await mesh("unassign laptop networking");
|
||||||
await mesh("push anchor");
|
await mesh("push anchor");
|
||||||
await new Promise((r) => setTimeout(r, 15_000));
|
await new Promise((r) => setTimeout(r, 15_000));
|
||||||
@@ -1533,14 +1408,13 @@ test("every name under a machine resolves to that machine", {
|
|||||||
`the machine that stayed lost its own name:\n${after}`);
|
`the machine that stayed lost its own name:\n${after}`);
|
||||||
|
|
||||||
await mesh("assign laptop networking");
|
await mesh("assign laptop networking");
|
||||||
await mesh("unassign anchor mesh-resolver");
|
await mesh("unassign anchor dnsmasq");
|
||||||
await mesh("push");
|
await mesh("push");
|
||||||
await new Promise((r) => setTimeout(r, 15_000));
|
await new Promise((r) => setTimeout(r, 15_000));
|
||||||
});
|
});
|
||||||
|
|
||||||
test("a service is reached by a name under the machine it runs on", {
|
test("a service is reached by a name under the machine it runs on", {
|
||||||
skip: skip || (!moduleExamples ? "set MESH_LAB_MODULES to mesh-controller's examples/modules" : false),
|
skip, timeout: 900_000,
|
||||||
timeout: 900_000,
|
|
||||||
}, async () => {
|
}, async () => {
|
||||||
// postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
|
// postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
|
||||||
// the node, so anything under a node's name must resolve to that node. What routes it once it
|
// the node, so anything under a node's name must resolve to that node. What routes it once it
|
||||||
@@ -1552,12 +1426,7 @@ test("a service is reached by a name under the machine it runs on", {
|
|||||||
// /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
|
// /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
|
||||||
// refusal rather than a fight over the file — and picking the wrong one here would have been
|
// refusal rather than a fight over the file — and picking the wrong one here would have been
|
||||||
// testing that fight.
|
// testing that fight.
|
||||||
for (const name of ["dnsmasq", "resolved-split-dns"]) {
|
await resolverModules(["anchor", "laptop"]);
|
||||||
const manifest = readFileSync(`${moduleExamples}/${name}.json`, "utf8");
|
|
||||||
await must("anchor", `cat > /tmp/${name}.json <<'MANIFEST'\n${manifest}\nMANIFEST`);
|
|
||||||
await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
|
||||||
await mesh(`module add /${name}.json`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both machines, because a node resolves from its own copy — the same rule as everything else
|
// Both machines, because a node resolves from its own copy — the same rule as everything else
|
||||||
// it holds. A mesh where one machine answers for all of them stops resolving when that machine
|
// it holds. A mesh where one machine answers for all of them stops resolving when that machine
|
||||||
@@ -1671,7 +1540,7 @@ test("a third-party workload is adopted, with the credential it already had", {
|
|||||||
const password = "the-password-it-already-had";
|
const password = "the-password-it-already-had";
|
||||||
|
|
||||||
await must("anchor", `printf %s ${quote(JSON.stringify({
|
await must("anchor", `printf %s ${quote(JSON.stringify({
|
||||||
module: "umami",
|
module: "adopted-analytics",
|
||||||
version: "1",
|
version: "1",
|
||||||
capabilities: ["container-runtime"],
|
capabilities: ["container-runtime"],
|
||||||
"own-secrets": {
|
"own-secrets": {
|
||||||
@@ -1707,13 +1576,13 @@ test("a third-party workload is adopted, with the credential it already had", {
|
|||||||
// seals it and cannot read it again. Given whole, as the environment lines the containers read.
|
// seals it and cannot read it again. Given whole, as the environment lines the containers read.
|
||||||
await must("anchor",
|
await must("anchor",
|
||||||
`printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
|
`printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
|
||||||
`docker exec -i mesh-controller /mesh-controller secret accept anchor umami database --from -`);
|
`docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics database --from -`);
|
||||||
await must("anchor",
|
await must("anchor",
|
||||||
`printf %s ${quote(
|
`printf %s ${quote(
|
||||||
`DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
|
`DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
|
||||||
`docker exec -i mesh-controller /mesh-controller secret accept anchor umami app --from -`);
|
`docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics app --from -`);
|
||||||
|
|
||||||
await mesh("assign anchor umami");
|
await mesh("assign anchor adopted-analytics");
|
||||||
await mesh("push anchor", 300_000);
|
await mesh("push anchor", 300_000);
|
||||||
|
|
||||||
// Both containers, and the network they share.
|
// Both containers, and the network they share.
|
||||||
@@ -1784,305 +1653,12 @@ test("a third-party workload is adopted, with the credential it already had", {
|
|||||||
// Running them needs their images stocked and two provisioners built, which is a separate and
|
// Running them needs their images stocked and two provisioners built, which is a separate and
|
||||||
// larger job. This is the half that can be known now, and it is the half where a design fault
|
// larger job. This is the half that can be known now, and it is the half where a design fault
|
||||||
// would live.
|
// would live.
|
||||||
test("the real modules resolve together, and compose a declaration a host accepts", {
|
// Three tests lived here that read the mesh's example modules, which moved to the catalogue.
|
||||||
skip, timeout: 300_000,
|
// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood
|
||||||
}, async (t) => {
|
// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the
|
||||||
// Everything the catalogue holds, except two whose names this mesh is already running under:
|
// catalogue's modules together and its gate is the composed declaration accepted and every core
|
||||||
// `registry` is the artifact store the suite stood up, and `umami` is the adopted workload —
|
// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates
|
||||||
// adding the catalogue's manifests would replace the records of modules that are live and
|
// on gitea running but does not yet ask it to answer on its port with the credential it was given,
|
||||||
// assigned, and the adopted umami would suddenly require a database it never asked for.
|
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
|
||||||
const modules = ["postgres", "keycloak", "gitea", "minio", "mailu",
|
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
|
||||||
"redis", "grafana", "nextcloud", "searxng", "influxdb", "verdaccio"];
|
// mesh-grant-end-to-end, against the catalogue's redis.
|
||||||
const planned: string[] = [];
|
|
||||||
for (const name of modules) {
|
|
||||||
const raw = readFileSync(
|
|
||||||
`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
|
|
||||||
// Pointed at this scenario's registry before being added, exactly as the forge is.
|
|
||||||
//
|
|
||||||
// **Not cosmetic.** Some of these name an image the mesh builds, whose digest does not exist
|
|
||||||
// until it is built — so the file legitimately carries a placeholder, and composing a
|
|
||||||
// declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is
|
|
||||||
// what this test used to do.
|
|
||||||
const pinned = pinnedInto(raw, held);
|
|
||||||
// What this scenario does not serve cannot be redirected, and a module still naming a
|
|
||||||
// placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped
|
|
||||||
// and said, rather than silently dropped: a planning test quietly covering four modules
|
|
||||||
// instead of five is the false coverage this suite exists to prevent.
|
|
||||||
const left = stillUnpinned(pinned);
|
|
||||||
if (left.length > 0) {
|
|
||||||
console.log(`skipping ${name}: this scenario serves no ${left.join(", ")}`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
planned.push(name);
|
|
||||||
await must("anchor", `printf %s ${quote(pinned)} > /${name}.json`);
|
|
||||||
await must("anchor", `docker cp /${name}.json mesh-controller:/${name}.json`);
|
|
||||||
await mesh(`module add /${name}.json`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Put the machine back whatever happens.** Tests here share one mesh, so what this one
|
|
||||||
// assigns is what the next one inherits. Written at the end of the body once, it was skipped
|
|
||||||
// the first time this test failed — and the next test's push was refused by a module this one
|
|
||||||
// had left behind, which reads as a fault in the test that was actually working.
|
|
||||||
t.after(async () => {
|
|
||||||
for (const name of planned) await mesh(`unassign anchor ${name}`).catch(() => {});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Assigned one at a time, because assignment resolves the whole set and says so immediately.
|
|
||||||
// A refusal here is the graph rejecting something, which is the point of asking.
|
|
||||||
for (const name of planned) {
|
|
||||||
await mesh(`assign anchor ${name}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const plan = await mesh("plan anchor --json", 120_000);
|
|
||||||
const declaration = JSON.parse(plan.slice(plan.indexOf("{")));
|
|
||||||
const byId = new Map<string, any>(
|
|
||||||
(declaration.resources as any[]).map((r) => [r.id, r]));
|
|
||||||
const ids = [...byId.keys()];
|
|
||||||
|
|
||||||
// Every module's own network, which only exists because more than one container needs to reach
|
|
||||||
// another by name.
|
|
||||||
for (const id of ["postgres.net", "keycloak.net", "minio.net", "mailu.net"]) {
|
|
||||||
assert.ok(byId.has(id), `${id} is missing; ${ids.length} resources: ${ids.join(", ")}`);
|
|
||||||
assert.equal(byId.get(id).type, "network");
|
|
||||||
}
|
|
||||||
|
|
||||||
// The cross-module edge: keycloak asked for a database and was told where it is and given a
|
|
||||||
// credential. Neither file is anything keycloak's manifest could have written.
|
|
||||||
const bound = [...byId.values()].find((r) =>
|
|
||||||
r.type === "file" && r.path === "/var/lib/keycloak/database.json");
|
|
||||||
assert.ok(bound, `keycloak was never told where its database is: ${ids.join(", ")}`);
|
|
||||||
assert.match(JSON.stringify(bound), /postgres/,
|
|
||||||
"keycloak's binding does not name what answered its requirement");
|
|
||||||
|
|
||||||
// The password, alone in a file and sealed. It is a password and nothing else, so nothing reads
|
|
||||||
// it as configuration — novox/hq 04-ISSUES/023 and the playbook both turn on that distinction.
|
|
||||||
const credential = [...byId.values()].find((r) =>
|
|
||||||
r.type === "file" && r.path === "/var/lib/keycloak/database.secret");
|
|
||||||
assert.ok(credential, `keycloak was given no credential for its database: ${ids.join(", ")}`);
|
|
||||||
assert.ok(credential.sealed, "keycloak's credential is not sealed, so the mesh can read it");
|
|
||||||
assert.ok(!credential.content, "a credential arrived as content rather than sealed");
|
|
||||||
|
|
||||||
// And the connection itself, which keycloak could not have written: the address and port come
|
|
||||||
// from what the provider serves, and the user name from what the mesh decided both ends would
|
|
||||||
// call this consumer (novox/hq 04-ISSUES/023).
|
|
||||||
const connection = [...byId.values()].find((r) =>
|
|
||||||
r.type === "file" && r.path === "/var/lib/keycloak/database.env");
|
|
||||||
assert.ok(connection, "keycloak was given no database configuration");
|
|
||||||
assert.match(connection.content, /KC_DB_USERNAME=mesh_[a-z0-9_]+_keycloak/,
|
|
||||||
`keycloak was not told what name to present:\n${connection.content}`);
|
|
||||||
assert.doesNotMatch(connection.content, /\$\{bound:/,
|
|
||||||
`a placeholder reached the machine as a value:\n${connection.content}`);
|
|
||||||
|
|
||||||
// The password is the one hole left open, and the sealed value travels beside it. The mesh
|
|
||||||
// discarded the plaintext, so the host is the only thing that can close it.
|
|
||||||
assert.match(connection.content, /KC_DB_PASSWORD=\$\{secret:postgres-database\}/,
|
|
||||||
`the password was not left for the host to fill:\n${connection.content}`);
|
|
||||||
assert.ok(connection.secrets?.["postgres-database"],
|
|
||||||
"the sealed credential did not travel with the file that needs it");
|
|
||||||
assert.doesNotMatch(JSON.stringify(connection.content), /postgres-database":"[A-Za-z0-9+/]{24,}/,
|
|
||||||
"the credential was written into the configuration in the clear");
|
|
||||||
|
|
||||||
// And the provider was told who asked, which is what its provisioner reconciles against.
|
|
||||||
const grants = [...byId.values()].find((r) =>
|
|
||||||
r.type === "file" && String(r.path).startsWith("/var/lib/postgres/grants"));
|
|
||||||
assert.ok(grants, "postgres was never told which modules were granted a database");
|
|
||||||
assert.match(JSON.stringify(grants), /keycloak|gitea/,
|
|
||||||
"the grants file names neither module that asked for a database");
|
|
||||||
|
|
||||||
// Secrets reach containers as files, never as environment in the declaration.
|
|
||||||
const containers = [...byId.values()].filter((r) => r.type === "container");
|
|
||||||
assert.ok(containers.length >= 12,
|
|
||||||
`only ${containers.length} containers; mailu alone is nine`);
|
|
||||||
for (const c of containers) {
|
|
||||||
for (const [key, value] of Object.entries(c.env ?? {})) {
|
|
||||||
// **An absolute path is a reference to a secret, not a secret**, and naming one is the
|
|
||||||
// whole design: the mesh delivers a credential as a file and a module says where.
|
|
||||||
//
|
|
||||||
// Excluded because `/` is in the base64 alphabet, so any path of 24 characters or more
|
|
||||||
// matched — `MESH_BROKER_FILE=/var/lib/mesh/builder/broker` was reported as a credential
|
|
||||||
// the broker would see. A check that fires on the right shape for the wrong reason is
|
|
||||||
// worse than none: it is the one that gets suppressed, and then it is not there when it
|
|
||||||
// is right.
|
|
||||||
if (String(value).startsWith("/")) continue;
|
|
||||||
assert.doesNotMatch(String(value), /^[A-Za-z0-9+/]{24,}={0,2}$/,
|
|
||||||
`${c.name} carries something secret-shaped in env.${key}, which the broker would see`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
// The first of the real module descriptions to actually run.
|
|
||||||
//
|
|
||||||
// **Everything before this stopped at composing a declaration.** That proves the control plane and
|
|
||||||
// the host agree, and proves nothing about whether the thing described works — which is how five
|
|
||||||
// modules sat pinned to images that did not exist, parsing and resolving perfectly
|
|
||||||
// (novox/hq 04-ISSUES/025).
|
|
||||||
//
|
|
||||||
// The forge is the one worth running first. It needs a database from another module, a password it
|
|
||||||
// did not choose, and a connection string it could not have written itself: the address and port
|
|
||||||
// come from what the database serves, and the user name from what the mesh decided both ends would
|
|
||||||
// call it (04-ISSUES/022 and 023). If any of that is wrong it cannot start, and nothing else in
|
|
||||||
// this file would notice.
|
|
||||||
test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 }, async () => {
|
|
||||||
for (const name of ["postgres", "gitea"]) {
|
|
||||||
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
|
|
||||||
// An image the mesh builds has no digest until it is built, and one it does not build belongs
|
|
||||||
// to whichever registry served it. Only the first is rewritten; the second is pulled.
|
|
||||||
const pinned = pinnedInto(raw, held);
|
|
||||||
assert.deepEqual(stillUnpinned(pinned), [],
|
|
||||||
`${name} still names an image nothing serves, so it could not start`);
|
|
||||||
await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`);
|
|
||||||
await must("anchor", `docker cp /run-${name}.json mesh-controller:/run-${name}.json`);
|
|
||||||
await mesh(`module add /run-${name}.json`);
|
|
||||||
await mesh(`assign anchor ${name}`);
|
|
||||||
}
|
|
||||||
await mesh("push anchor", 300_000);
|
|
||||||
|
|
||||||
// **What the machine says it did, before asking what it produced.** This test pushed and then
|
|
||||||
// waited for a database role, so when the containers were never created at all it reported "no
|
|
||||||
// login was created" — true, and silent about the reason. A push that was accepted and an apply
|
|
||||||
// that worked are different facts, and the second is the one this depends on.
|
|
||||||
//
|
|
||||||
// And waited for, because `push` sends without waiting. Reading `status` the instant it returns
|
|
||||||
// describes the apply *before* this one, which is how this test came to report a missing
|
|
||||||
// container while insisting the machine was fine.
|
|
||||||
await settled("anchor");
|
|
||||||
const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out;
|
|
||||||
// Named with what the mesh meant to send, not only with what the machine has. A container that
|
|
||||||
// is absent because the mesh never asked for it and one that is absent because the machine could
|
|
||||||
// not make it are the same sentence here and different faults entirely, and the plan is the only
|
|
||||||
// thing that tells them apart.
|
|
||||||
assert.match(running, /\bpostgres\b/,
|
|
||||||
`the database module was pushed and no container for it exists:\n${running}\n\n` +
|
|
||||||
`what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` +
|
|
||||||
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
|
||||||
|
|
||||||
// The database first: until the provisioner has made the login, the forge has nothing to
|
|
||||||
// connect to and its own start would prove only that it retries.
|
|
||||||
const psql = async (q: string) =>
|
|
||||||
(await on("anchor",
|
|
||||||
`docker exec postgres psql -U postgres -qAt -c ${quote(q)}`, 60_000)).out.trim();
|
|
||||||
|
|
||||||
// Both halves in one poll. The provisioner makes the role and then the database, and a test
|
|
||||||
// that waited for the first and checked the second once was racing the gap between two
|
|
||||||
// statements — it lost, once, eighteen seconds into a run.
|
|
||||||
let made = "";
|
|
||||||
for (let i = 0; i < 40 && made !== "t"; i++) {
|
|
||||||
made = await psql("select true from pg_roles where rolname = 'mesh_anchor_gitea'" +
|
|
||||||
" and exists (select from pg_database where datname = 'gitea')");
|
|
||||||
if (made !== "t") await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.equal(made, "t",
|
|
||||||
`no login was created for the forge:\n${(await on("anchor", "docker logs mesh-provision-postgres 2>&1 | tail -20")).out}`);
|
|
||||||
|
|
||||||
// And the forge itself, answering. Not that its container exists — that it serves.
|
|
||||||
//
|
|
||||||
// On the port the mesh assigned, not the one the module declared (novox/hq ADR 0038): the
|
|
||||||
// module says 3000 and the machine publishes wherever the mesh put it. Read from the plan,
|
|
||||||
// because the plan is the same composition a push sends.
|
|
||||||
const planned = await mesh("plan anchor --json", 120_000);
|
|
||||||
const mapping = (JSON.parse(planned.slice(planned.indexOf("{"))).resources as any[])
|
|
||||||
.find((r) => r.id === "gitea.server")?.ports
|
|
||||||
?.map(String).find((p: string) => p.endsWith(":3000"));
|
|
||||||
assert.ok(mapping, "the plan does not say where the machine publishes the forge");
|
|
||||||
const at = mapping.split(":")[0];
|
|
||||||
let answered = false;
|
|
||||||
let said = { out: "", ok: false };
|
|
||||||
for (let i = 0; i < 60 && !answered; i++) {
|
|
||||||
said = await on("anchor", `curl -sf -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${at}/`, 30_000);
|
|
||||||
answered = said.out.trim().startsWith("2") || said.out.trim() === "303";
|
|
||||||
if (!answered) await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
assert.ok(answered,
|
|
||||||
`the forge never answered (last: ${said.out.trim()}):\n` +
|
|
||||||
`${(await on("anchor", "docker logs gitea 2>&1 | tail -25")).out}`);
|
|
||||||
|
|
||||||
// **The credential actually worked.** A forge that started and could not reach its database
|
|
||||||
// would still answer on its port, so the log is where the difference lives.
|
|
||||||
const log = (await on("anchor", "docker logs gitea 2>&1 | tail -60")).out;
|
|
||||||
assert.doesNotMatch(log, /password authentication failed|connection refused|does not exist/i,
|
|
||||||
`the forge started and could not use the database it was given:\n${log}`);
|
|
||||||
|
|
||||||
await mesh("unassign anchor gitea");
|
|
||||||
await mesh("unassign anchor postgres");
|
|
||||||
await mesh("push anchor", 300_000);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600_000 }, async (t) => {
|
|
||||||
// The third provision after a database and a bucket, and the first whose tenancy is enforced
|
|
||||||
// by the store's own ACL rather than by separate namespaces: every consumer shares one
|
|
||||||
// keyspace, so the grant is a pattern — and the test is that the pattern means what the
|
|
||||||
// manifest said, in both directions.
|
|
||||||
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8");
|
|
||||||
const pinned = pinnedInto(raw, held);
|
|
||||||
assert.deepEqual(stillUnpinned(pinned), [],
|
|
||||||
"redis still names an image nothing serves, so it could not start");
|
|
||||||
await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`);
|
|
||||||
await must("anchor", `docker cp /run-redis.json mesh-controller:/run-redis.json`);
|
|
||||||
await mesh("module add /run-redis.json");
|
|
||||||
|
|
||||||
// A consumer with no container: what is under test is the credential's reach, and files on the
|
|
||||||
// machine are enough to prove it — the same reduction the first credential test makes.
|
|
||||||
await must("anchor", `printf %s '{"module":"cachetest","version":"1",` +
|
|
||||||
`"requires":["redis-cache"],` +
|
|
||||||
`"contributes":{"redis-cache":{"prefix":"cachetest"}},` +
|
|
||||||
`"binds":{"redis-cache":"/var/lib/cachetest/cache.json"},` +
|
|
||||||
`"secrets":{"redis-cache":"/var/lib/cachetest/cache.secret"},` +
|
|
||||||
`"resources":[{"id":"state","type":"directory","path":"/var/lib/cachetest","mode":"0700"}]}' ` +
|
|
||||||
`> /cachetest.json`);
|
|
||||||
await must("anchor", `docker cp /cachetest.json mesh-controller:/cachetest.json`);
|
|
||||||
await mesh("module add /cachetest.json");
|
|
||||||
|
|
||||||
await mesh("assign anchor redis");
|
|
||||||
await mesh("assign anchor cachetest");
|
|
||||||
await mesh("push anchor", 300_000);
|
|
||||||
await settled("anchor");
|
|
||||||
|
|
||||||
t.after(async () => {
|
|
||||||
for (const name of ["cachetest", "redis"]) {
|
|
||||||
await mesh(`unassign anchor ${name}`).catch(() => {});
|
|
||||||
}
|
|
||||||
await mesh("push anchor", 300_000).catch(() => {});
|
|
||||||
});
|
|
||||||
|
|
||||||
// What the mesh told each end. The consumer's user name comes from its binding; the user's
|
|
||||||
// password from the sealed file beside it — both written by the host, neither invented here.
|
|
||||||
const bound = JSON.parse(await must("anchor", `cat /var/lib/cachetest/cache.json`));
|
|
||||||
const user = bound.as;
|
|
||||||
assert.ok(user?.startsWith("mesh_"), `the binding does not carry a usable user: ${user}`);
|
|
||||||
const secret = (await must("anchor", `cat /var/lib/cachetest/cache.secret`)).trim();
|
|
||||||
|
|
||||||
// The provisioner has to have run before anything can authenticate. Waited for via the store
|
|
||||||
// itself: the user list, asked with the server's own password, which the conf file the host
|
|
||||||
// wrote holds on the machine.
|
|
||||||
const admin = (await must("anchor",
|
|
||||||
`awk '/^requirepass/ {print $2}' /var/lib/redis-module/redis.conf`)).trim();
|
|
||||||
let granted = false;
|
|
||||||
for (let i = 0; i < 40 && !granted; i++) {
|
|
||||||
const users = (await on("anchor",
|
|
||||||
`docker exec redis redis-cli --no-auth-warning -a ${quote(admin)} ACL USERS`)).out;
|
|
||||||
granted = users.includes(user);
|
|
||||||
if (!granted) await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.ok(granted, `no user was created for the consumer:
|
|
||||||
` +
|
|
||||||
`containers:\n${(await on("anchor", "docker ps -a --format '{{.Names}} {{.Status}}' | head -20")).out}\n` +
|
|
||||||
`the store:\n${(await on("anchor", "docker logs redis 2>&1 | tail -15")).out}\n` +
|
|
||||||
`the provisioner:\n${(await on("anchor", "docker logs mesh-provision-redis 2>&1 | tail -15")).out}`);
|
|
||||||
|
|
||||||
const asConsumer = (command: string) =>
|
|
||||||
on("anchor", `docker exec redis redis-cli --no-auth-warning ` +
|
|
||||||
`--user ${quote(user)} --pass ${quote(secret)} ${command}`);
|
|
||||||
|
|
||||||
// Its own keys: usable.
|
|
||||||
assert.match((await asConsumer("SET cachetest:proof yes")).out, /OK/,
|
|
||||||
"the consumer cannot write under the prefix it was granted");
|
|
||||||
assert.match((await asConsumer("GET cachetest:proof")).out, /yes/,
|
|
||||||
"the consumer cannot read back what it wrote");
|
|
||||||
|
|
||||||
// Anyone else's: refused by the store itself, which is the entire point of the grant.
|
|
||||||
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
|
|
||||||
"the consumer wrote outside its prefix — the grant means more than the manifest said");
|
|
||||||
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
|
|
||||||
"the consumer can flush the store, which no tenant may");
|
|
||||||
});
|
|
||||||
|
|||||||
@@ -1,252 +0,0 @@
|
|||||||
/**
|
|
||||||
* The whole grant for an S3 bucket, mesh-driven — novox/hq ADR 0052/0053, the minio case.
|
|
||||||
*
|
|
||||||
* The postgres bed proves the provider/consumer contract for a database. This proves it for object
|
|
||||||
* storage, on a provider whose code drives the `mc` CLI (so the runtime image carries it): minio is
|
|
||||||
* assigned, a consumer that requires s3-bucket is assigned, and the mesh mints one secret key, sealing
|
|
||||||
* a copy to each end. minio's provisioner — reading only the mesh's contributions — creates a bucket
|
|
||||||
* and a service account under the access key the mesh derived, with the secret it minted. The proof is
|
|
||||||
* the consumer reaching its bucket with the access key and secret the mesh delivered it. Nothing is
|
|
||||||
* placed by the test.
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh minio builds mesh-runtime-minio:development (with mc), which
|
|
||||||
* scenarios/minio-node.yml stocks. minio/minio:latest must be in the local daemon.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
// 04-ISSUES/010 is fixed by ADR 0054: an S3 access key is capped at 20, and the mesh derives
|
|
||||||
// `mesh_<node>_<module>`, so `bucketuser` on `anchor` (22) would overflow — but a consumer declares a
|
|
||||||
// short `slug` and its identity fits. This bed's consumer does exactly that.
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "minio-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Same rule minio's client uses to name a bucket for a consumer — recomputed so the test knows it. */
|
|
||||||
function bucketFor(as: string): string {
|
|
||||||
const name = as.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 63);
|
|
||||||
return name.length >= 3 ? name : `mesh-${name}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("the mesh grants a consumer an S3 bucket, and the credential it delivers reaches it", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// The PROVIDER: minio in its committed shape — server and a broker-bound runtime (carrying mc) on
|
|
||||||
// the private minio network, the runtime running the provisioner. No published port on this
|
|
||||||
// single-node bed; the consumer reaches minio over the private network by name.
|
|
||||||
const minioManifest = JSON.stringify({
|
|
||||||
module: "minio",
|
|
||||||
version: "1",
|
|
||||||
provides: [{ name: "s3-bucket", scope: "mesh" }],
|
|
||||||
serves: { "s3-bucket": { scheme: "http", region: "us-east-1" } },
|
|
||||||
emits: ["module.minio.bucket.created", "module.minio.bucket.removed"],
|
|
||||||
receives: { "s3-bucket": "/var/lib/minio/grants/mesh.json" },
|
|
||||||
grants: { "s3-bucket": "/var/lib/minio/grants" },
|
|
||||||
"own-secrets": { root: "/var/lib/minio/root.secret", broker: "/var/lib/mesh/minio/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/minio", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/minio", mode: "0700" },
|
|
||||||
{ id: "grants", type: "directory", path: "/var/lib/minio/grants", mode: "0700" },
|
|
||||||
{ id: "root-env", type: "file", path: "/var/lib/minio/root.env", mode: "0600", content: "MINIO_ROOT_USER=meshroot\nMINIO_ROOT_PASSWORD=${secret:root}\n" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/minio/data/data1-1", mode: "0700" },
|
|
||||||
{ id: "net", type: "network", name: "minio" },
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "minio", image: pinned("minio/minio"), network: "minio",
|
|
||||||
args: ["server", "/data", "--console-address", ":9001"],
|
|
||||||
"env-file": ["/var/lib/minio/root.env"],
|
|
||||||
volumes: ["/services/minio/data/data1-1:/data"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-minio", image: pinned("mesh-runtime-minio"),
|
|
||||||
network: "minio",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/minio/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/minio/grants:/var/lib/minio/grants:ro",
|
|
||||||
"/var/lib/minio/root.secret:/run/secrets/root:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_MINIO_ENDPOINT: "http://minio:9000",
|
|
||||||
MESH_MINIO_ROOT_USER: "meshroot",
|
|
||||||
MESH_MINIO_ROOT_PASSWORD_FILE: "/run/secrets/root",
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/minio/grants/mesh.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const consumerManifest = JSON.stringify({
|
|
||||||
module: "bucketuser",
|
|
||||||
version: "1",
|
|
||||||
// A short slug, so the derived identity `mesh_anchor_bkt` fits an S3 access key's 20 chars where
|
|
||||||
// `mesh_anchor_bucketuser` (22) would not (novox/hq ADR 0054, 04-ISSUES/010).
|
|
||||||
slug: "bkt",
|
|
||||||
requires: ["s3-bucket"],
|
|
||||||
contributes: { "s3-bucket": { name: "bucketuser" } },
|
|
||||||
binds: { "s3-bucket": "/var/lib/bucketuser/s3.json" },
|
|
||||||
secrets: { "s3-bucket": "/var/lib/bucketuser/s3.secret" },
|
|
||||||
resources: [{ id: "state", type: "directory", path: "/var/lib/bucketuser", mode: "0700" }],
|
|
||||||
});
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(minioManifest)} > /tmp/minio.json && docker cp /tmp/minio.json mesh-controller:/minio.json`);
|
|
||||||
await mesh("module add /minio.json");
|
|
||||||
await mesh(`module issue minio --node ${MACHINE}`);
|
|
||||||
await mesh(`assign ${MACHINE} minio`);
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(consumerManifest)} > /tmp/bucketuser.json && docker cp /tmp/bucketuser.json mesh-controller:/bucketuser.json`);
|
|
||||||
await mesh("module add /bucketuser.json");
|
|
||||||
await mesh(`assign ${MACHINE} bucketuser`);
|
|
||||||
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
// The mesh delivered the consumer its bound file and its unsealed secret.
|
|
||||||
let boundRaw = "";
|
|
||||||
const untilBound = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilBound) {
|
|
||||||
const got = await on(`cat /var/lib/bucketuser/s3.json 2>/dev/null`);
|
|
||||||
if (got.ok && /"as"/.test(got.out)) { boundRaw = got.out; break; }
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(boundRaw, /"as"/, `the consumer was never told about its bucket:\n${boundRaw}`);
|
|
||||||
const bound = JSON.parse(boundRaw) as { as: string; provision: string };
|
|
||||||
assert.equal(bound.provision, "s3-bucket");
|
|
||||||
const accessKey = bound.as;
|
|
||||||
const secretKey = (await must(`cat /var/lib/bucketuser/s3.secret`)).trim();
|
|
||||||
assert.ok(accessKey && secretKey, `the consumer's access key or secret was empty (as=${accessKey})`);
|
|
||||||
const bucket = bucketFor(accessKey);
|
|
||||||
|
|
||||||
// THE PROOF: reach the bucket as the consumer, with the access key and secret the mesh delivered
|
|
||||||
// it. mc listing the consumer's own bucket means the service account, the bucket, and the secret all
|
|
||||||
// line up across the two ends. A provisioner that set a different secret answers "Access Denied".
|
|
||||||
const probe =
|
|
||||||
`mc alias set probe http://minio:9000 ${quote(accessKey)} ${quote(secretKey)} >/dev/null 2>&1 && ` +
|
|
||||||
`mc ls probe/${quote(bucket)}/`;
|
|
||||||
let out = { out: "", ok: false };
|
|
||||||
const untilReach = Date.now() + 90_000;
|
|
||||||
while (Date.now() < untilReach) {
|
|
||||||
out = await on(`docker exec mesh-minio sh -c ${quote(probe)} 2>&1`);
|
|
||||||
if (out.ok) break;
|
|
||||||
if (/denied/i.test(out.out)) break; // fast-fail: the credential is wrong
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.doesNotMatch(out.out, /denied/i,
|
|
||||||
`the consumer could not reach its bucket with the mesh's secret — the two ends do not agree:\n${out.out}`);
|
|
||||||
assert.ok(out.ok,
|
|
||||||
`the consumer could not list its granted bucket ${bucket} as ${accessKey}:\n${out.out}\n---\n${(await on(`docker logs mesh-minio 2>&1 | tail -30`)).out}`);
|
|
||||||
});
|
|
||||||
@@ -24,7 +24,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -37,7 +37,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "openai-bed";
|
const SCENARIO = "openai-bed";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -156,7 +156,6 @@ after(async () => {
|
|||||||
test("a static-key model-access licence delivers the operator's API key to the consumer, unchanged", {
|
test("a static-key model-access licence delivers the operator's API key to the consumer, unchanged", {
|
||||||
skip, timeout: 1_500_000,
|
skip, timeout: 1_500_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
const consumerImage = pinned("mesh-runtime-openai-consumer");
|
|
||||||
|
|
||||||
// --- the licence, a record with vendor openai (static-key) -------------------------------------
|
// --- the licence, a record with vendor openai (static-key) -------------------------------------
|
||||||
// No manager: a static-key licence has none (mesh-controller refuses `licence manager` on it). The
|
// No manager: a static-key licence has none (mesh-controller refuses `licence manager` on it). The
|
||||||
@@ -172,36 +171,15 @@ test("a static-key model-access licence delivers the operator's API key to the c
|
|||||||
await mesh(`licence key personal --file /openai-key`);
|
await mesh(`licence key personal --file /openai-key`);
|
||||||
|
|
||||||
// --- deploy the consumer -----------------------------------------------------------------------
|
// --- deploy the consumer -----------------------------------------------------------------------
|
||||||
// Inline manifest mirroring the committed module.json: a model-access holder whose delivered key
|
// The catalogue's own manifest (novox/hq 04-ISSUES/073): a model-access holder whose delivered key
|
||||||
// arrives at its secret path. The scheduled apply container installs as present state (ADR 0053) and
|
// arrives at its secret path. The scheduled apply container installs as present state (ADR 0053) and
|
||||||
// its image is pulled at apply (schedule-pull); the test drives apply directly for a deterministic
|
// its image is pulled at apply (schedule-pull); the test drives apply directly for a deterministic
|
||||||
// flow rather than waiting on cron.
|
// flow rather than waiting on cron.
|
||||||
const consumerManifest = JSON.stringify({
|
const consumerManifest = catalogueModule("openai-consumer", held);
|
||||||
module: "openai-consumer",
|
|
||||||
version: "1",
|
|
||||||
requires: ["model-access"],
|
|
||||||
binds: { "model-access": "/var/lib/openai-consumer/model.json" },
|
|
||||||
secrets: { "model-access": "/var/lib/openai-consumer/api-key" },
|
|
||||||
resources: [
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/openai-consumer", mode: "0700" },
|
|
||||||
{ id: "config", type: "directory", path: "/var/lib/openai-consumer/config", mode: "0700" },
|
|
||||||
{
|
|
||||||
id: "apply", type: "container", name: "mesh-openai-consumer-apply",
|
|
||||||
image: consumerImage, network: "host", schedule: "*/5 * * * *",
|
|
||||||
args: ["run", "/app/modules/openai-consumer/dist/apply/index.js"],
|
|
||||||
volumes: ["/var/lib/openai-consumer:/run/state"],
|
|
||||||
env: {
|
|
||||||
MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/api-key",
|
|
||||||
MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
|
|
||||||
MESH_OPENAI_ENV_FILE: "/run/state/config/openai.env",
|
|
||||||
MESH_OPENAI_CREDENTIALS_FILE: "/run/state/config/auth.json",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(consumerManifest)} > /tmp/openai-consumer.json && docker cp /tmp/openai-consumer.json mesh-controller:/openai-consumer.json`);
|
await must(`printf %s ${quote(consumerManifest)} > /tmp/openai-consumer.json && docker cp /tmp/openai-consumer.json mesh-controller:/openai-consumer.json`);
|
||||||
await mesh(`module add /openai-consumer.json`);
|
await mesh(`module add /openai-consumer.json`);
|
||||||
await mesh(`module issue openai-consumer --node ${MACHINE}`);
|
// No `module issue`: the consumer speaks on no bus, and issuing a module with no broker secret
|
||||||
|
// to deliver into is refused (novox/hq issue 078).
|
||||||
await mesh(`assign ${MACHINE} openai-consumer`);
|
await mesh(`assign ${MACHINE} openai-consumer`);
|
||||||
await mesh(`push ${MACHINE}`);
|
await mesh(`push ${MACHINE}`);
|
||||||
await settled();
|
await settled();
|
||||||
|
|||||||
@@ -1,241 +0,0 @@
|
|||||||
/**
|
|
||||||
* The whole grant for a database, mesh-driven — novox/hq ADR 0052/0053, the postgres case.
|
|
||||||
*
|
|
||||||
* The redis bed proves the provider/consumer contract for a cache. This proves it for a database, on
|
|
||||||
* a provider whose code shells out to `psql` (so the runtime image carries it): postgres is assigned,
|
|
||||||
* a consumer that requires postgres-database is assigned, and the mesh mints one password, seals a
|
|
||||||
* copy to each end, and writes each its file. postgres's provisioner — reading only the mesh's
|
|
||||||
* contributions — creates a role and a database under the login the mesh derived, with the password
|
|
||||||
* the mesh minted. The proof is the consumer connecting to its database with the credential the mesh
|
|
||||||
* delivered it. Nothing is placed by the test.
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh postgres builds mesh-runtime-postgres:development (with psql),
|
|
||||||
* which scenarios/postgres-node.yml stocks.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "postgres-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("the mesh grants a consumer a postgres database, and the credential it delivers connects", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// The PROVIDER: postgres in its committed shape — server and a broker-bound runtime (carrying psql)
|
|
||||||
// on the private postgres network, the runtime running the provisioner.
|
|
||||||
const postgresManifest = JSON.stringify({
|
|
||||||
module: "postgres",
|
|
||||||
version: "1",
|
|
||||||
provides: [{ name: "postgres-database", scope: "mesh" }],
|
|
||||||
serves: { "postgres-database": {} },
|
|
||||||
emits: ["module.postgres.database.provisioned", "module.postgres.database.deprovisioned"],
|
|
||||||
consumes: ["module.postgres.database.provisioned", "module.postgres.database.deprovisioned"],
|
|
||||||
receives: { "postgres-database": "/var/lib/postgres/grants/mesh.json" },
|
|
||||||
grants: { "postgres-database": "/var/lib/postgres/grants" },
|
|
||||||
"own-secrets": { superuser: "/var/lib/postgres/superuser.secret", broker: "/var/lib/mesh/postgres/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/postgres", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/postgres", mode: "0700" },
|
|
||||||
{ id: "grants", type: "directory", path: "/var/lib/postgres/grants", mode: "0700" },
|
|
||||||
{ id: "superuser-env", type: "file", path: "/var/lib/postgres/superuser.env", mode: "0600", content: "POSTGRES_PASSWORD=${secret:superuser}\n" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/postgres/db-data", mode: "0700" },
|
|
||||||
{ id: "net", type: "network", name: "postgres" },
|
|
||||||
{
|
|
||||||
// No published port here: the foundation's own store already holds host :5432 on this
|
|
||||||
// single-node bed, and the consumer reaches postgres over the private network by name. The
|
|
||||||
// committed manifest publishes it for cross-node consumers, which is a different node.
|
|
||||||
id: "server", type: "container", name: "postgres", image: pinned("postgres"), network: "postgres",
|
|
||||||
env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres" },
|
|
||||||
"env-file": ["/var/lib/postgres/superuser.env"],
|
|
||||||
volumes: ["/services/postgres/db-data:/var/lib/postgresql/data"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-postgres", image: pinned("mesh-runtime-postgres"),
|
|
||||||
network: "postgres",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/postgres/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/postgres/grants:/var/lib/postgres/grants:ro",
|
|
||||||
"/var/lib/postgres/superuser.secret:/run/secrets/superuser:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/postgres/grants/mesh.json",
|
|
||||||
MESH_PROVISION_POSTGRES: "postgres://postgres@postgres:5432/postgres?sslmode=disable",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/superuser",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
// The CONSUMER: a module that requires postgres-database and contributes a name so it asks.
|
|
||||||
const consumerManifest = JSON.stringify({
|
|
||||||
module: "dbuser",
|
|
||||||
version: "1",
|
|
||||||
requires: ["postgres-database"],
|
|
||||||
contributes: { "postgres-database": { name: "dbuser" } },
|
|
||||||
binds: { "postgres-database": "/var/lib/dbuser/db.json" },
|
|
||||||
secrets: { "postgres-database": "/var/lib/dbuser/db.secret" },
|
|
||||||
resources: [{ id: "state", type: "directory", path: "/var/lib/dbuser", mode: "0700" }],
|
|
||||||
});
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(postgresManifest)} > /tmp/postgres.json && docker cp /tmp/postgres.json mesh-controller:/postgres.json`);
|
|
||||||
await mesh("module add /postgres.json");
|
|
||||||
await mesh(`module issue postgres --node ${MACHINE}`);
|
|
||||||
await mesh(`assign ${MACHINE} postgres`);
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(consumerManifest)} > /tmp/dbuser.json && docker cp /tmp/dbuser.json mesh-controller:/dbuser.json`);
|
|
||||||
await mesh("module add /dbuser.json");
|
|
||||||
await mesh(`assign ${MACHINE} dbuser`);
|
|
||||||
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
// The mesh delivered the consumer its bound file and its unsealed password.
|
|
||||||
let boundRaw = "";
|
|
||||||
const untilBound = Date.now() + 60_000;
|
|
||||||
while (Date.now() < untilBound) {
|
|
||||||
const got = await on(`cat /var/lib/dbuser/db.json 2>/dev/null`);
|
|
||||||
if (got.ok && /"as"/.test(got.out)) { boundRaw = got.out; break; }
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(boundRaw, /"as"/, `the consumer was never told about its database:\n${boundRaw}`);
|
|
||||||
const bound = JSON.parse(boundRaw) as { as: string; provision: string };
|
|
||||||
assert.equal(bound.provision, "postgres-database");
|
|
||||||
const as = bound.as;
|
|
||||||
const password = (await must(`cat /var/lib/dbuser/db.secret`)).trim();
|
|
||||||
assert.ok(as && password, `the consumer's login or password was empty (as=${as})`);
|
|
||||||
|
|
||||||
// THE PROOF: connect to postgres as the consumer, with the login and password the mesh delivered
|
|
||||||
// it, to the database postgres's provisioner created — a real password-checked TCP connection (the
|
|
||||||
// runtime carries psql). A `1` back means the role, the database, and the password all line up
|
|
||||||
// across the two ends. A provisioner that set a different password answers "authentication failed".
|
|
||||||
const conn = `postgresql://${as}:${encodeURIComponent(password)}@postgres:5432/${as}?sslmode=disable`;
|
|
||||||
let out = { out: "", ok: false };
|
|
||||||
const untilConn = Date.now() + 90_000;
|
|
||||||
while (Date.now() < untilConn) {
|
|
||||||
out = await on(`docker exec mesh-postgres psql ${quote(conn)} -tAc 'select 1' 2>&1`);
|
|
||||||
if (out.ok && /^1$/m.test(out.out)) break;
|
|
||||||
if (/authentication failed/i.test(out.out)) break; // fast-fail: the credential is wrong
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.doesNotMatch(out.out, /authentication failed/i,
|
|
||||||
`the consumer could not authenticate with the mesh's password — the two ends do not agree:\n${out.out}`);
|
|
||||||
assert.match(out.out, /^1$/m,
|
|
||||||
`the consumer could not connect to its granted database as ${as}:\n${out.out}\n---\n${(await on(`docker logs mesh-postgres 2>&1 | tail -30`)).out}`);
|
|
||||||
});
|
|
||||||
@@ -9,7 +9,7 @@
|
|||||||
* is on the broker — none of which happens if it could not reach the broker from the bridge.
|
* is on the broker — none of which happens if it could not reach the broker from the bridge.
|
||||||
*
|
*
|
||||||
* This mirrors the redis committed manifest exactly (server on `redis` with a published port, runtime
|
* This mirrors the redis committed manifest exactly (server on `redis` with a published port, runtime
|
||||||
* on `redis`), where provider-uses-mesh-credential used host networking. If this is green, the
|
* on `redis`), where the earlier host-networked bed (since retired) took the shortcut. If this is green, the
|
||||||
* private-network shape is the one to roll out to every provider.
|
* private-network shape is the one to roll out to every provider.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@@ -20,7 +20,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -33,7 +33,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "redis-node";
|
const SCENARIO = "redis-node";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
@@ -136,54 +136,15 @@ after(async () => {
|
|||||||
test("redis's runtime, on the backend's private network, binds the broker and provisions with the mesh's credential", {
|
test("redis's runtime, on the backend's private network, binds the broker and provisions with the mesh's credential", {
|
||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
// Exactly the committed redis shape: a private `redis` network, the server on it with a published
|
// The catalogue's redis (novox/hq 04-ISSUES/074): a private `redis` network, the server on it
|
||||||
// port, and the runtime on it too — reaching redis by name and the broker by NAT.
|
// with a published port, and the runtime on it too — reaching redis by name and the broker by
|
||||||
const manifest = JSON.stringify({
|
// NAT. Its own password is a `secret` the vault provides, so the vault is installed beside it.
|
||||||
module: "redis",
|
const vaultManifest = catalogueModule("mesh-vault", held);
|
||||||
version: "1",
|
await must(`printf %s ${quote(vaultManifest)} > /tmp/mesh-vault.json && docker cp /tmp/mesh-vault.json mesh-controller:/mesh-vault.json`);
|
||||||
provides: [{ name: "redis-cache", scope: "mesh" }],
|
await mesh("module add /mesh-vault.json");
|
||||||
serves: { "redis-cache": {} },
|
await mesh(`module issue mesh-vault --node ${MACHINE}`);
|
||||||
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
await mesh(`assign ${MACHINE} mesh-vault`);
|
||||||
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
const manifest = catalogueModule("redis", held);
|
||||||
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
|
|
||||||
grants: { "redis-cache": "/var/lib/redis-module/grants" },
|
|
||||||
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
|
|
||||||
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
|
|
||||||
{
|
|
||||||
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
|
|
||||||
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
|
|
||||||
},
|
|
||||||
{ id: "net", type: "network", name: "redis" },
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "redis",
|
|
||||||
ports: ["6379"],
|
|
||||||
volumes: [
|
|
||||||
"/services/redis/data:/data",
|
|
||||||
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
|
|
||||||
],
|
|
||||||
args: ["/etc/redis/redis.conf"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
|
|
||||||
network: "redis",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/redis-module/grants:/var/lib/redis-module/grants:ro",
|
|
||||||
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
|
|
||||||
MESH_PROVISION_REDIS: "redis:6379",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
|
||||||
await mesh("module add /redis.json");
|
await mesh("module add /redis.json");
|
||||||
await mesh(`module issue redis --node ${MACHINE}`);
|
await mesh(`module issue redis --node ${MACHINE}`);
|
||||||
|
|||||||
@@ -1,236 +0,0 @@
|
|||||||
/**
|
|
||||||
* A provider creates the resource with the credential the mesh minted — novox/hq ADR 0048.
|
|
||||||
*
|
|
||||||
* The old provisioner generated its own password, sealed it with a key nothing delivered, and
|
|
||||||
* handed it back. This proves the corrected contract: redis's provisioner reads the mesh's
|
|
||||||
* contributions file and, for each consumer, the password the mesh minted and the host unsealed, and
|
|
||||||
* creates the ACL user under the login the mesh derived, with that exact password. No $MESH_SEAL_KEY
|
|
||||||
* is set anywhere. The proof is authentication: a client logging in as that consumer with the mesh's
|
|
||||||
* password gets PONG — where a provisioner that invented its own password would answer WRONGPASS.
|
|
||||||
*
|
|
||||||
* A hand-written contributions file and secret stand in for the control plane here (a full grant
|
|
||||||
* from a second module is a heavier bed); their SHAPE is exactly what mesh-controller writes — a
|
|
||||||
* `receives` doc with `as`/`secret`, and the secret file the host leaves after unsealing.
|
|
||||||
*
|
|
||||||
* MESH_LAB_HOST_BINARY=.../mesh-host MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
||||||
* scripts/build-module-runtime.sh redis builds mesh-runtime-redis:development, which
|
|
||||||
* scenarios/redis-node.yml stocks.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { test, before, after } from "node:test";
|
|
||||||
import assert from "node:assert/strict";
|
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
|
||||||
const binary = hostBinaryPath();
|
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
|
||||||
? `lab not usable: ${capability.why}`
|
|
||||||
: !binary || !existsSync(binary)
|
|
||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
||||||
: !bundle || !existsSync(bundle)
|
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
||||||
: false;
|
|
||||||
|
|
||||||
const SCENARIO = "redis-node";
|
|
||||||
const MACHINE = "anchor";
|
|
||||||
|
|
||||||
let instanceId = "";
|
|
||||||
let held: HeldImage[] = [];
|
|
||||||
|
|
||||||
function quote(s: string): string {
|
|
||||||
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
||||||
const { stdout } = await exec(instanceId, MACHINE, [
|
|
||||||
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
||||||
], timeoutMs);
|
|
||||||
const marker = stdout.lastIndexOf("__exit=");
|
|
||||||
if (marker < 0) return { out: stdout, ok: false };
|
|
||||||
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function must(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
const { out, ok } = await on(command, timeoutMs);
|
|
||||||
if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
||||||
return must(`docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
|
||||||
return foundationBundle(bundle, images);
|
|
||||||
}
|
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
|
||||||
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
||||||
assert.ok(found, `no token in:\n${said}`);
|
|
||||||
return found;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function settled(withinMs = 480_000): Promise<void> {
|
|
||||||
const until = Date.now() + withinMs;
|
|
||||||
let last = "";
|
|
||||||
while (Date.now() < until) {
|
|
||||||
const asked = await on(`docker exec mesh-controller /mesh-controller status --json`);
|
|
||||||
if (asked.ok) {
|
|
||||||
try {
|
|
||||||
const state = JSON.parse(asked.out) as {
|
|
||||||
wrong: { node: string; outcome: string }[];
|
|
||||||
waiting: { node: string }[];
|
|
||||||
reported: { node: string; outcome: string; current: boolean }[];
|
|
||||||
};
|
|
||||||
const bad = state.wrong.find((w) => w.node === MACHINE);
|
|
||||||
if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
|
|
||||||
const word = state.reported.find((r) => r.node === MACHINE);
|
|
||||||
if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
|
|
||||||
last = asked.out;
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof Error && err.message.includes("did not apply")) throw err;
|
|
||||||
last = asked.out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await new Promise((r) => setTimeout(r, 5000));
|
|
||||||
}
|
|
||||||
throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
before(async () => {
|
|
||||||
if (skip) return;
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
|
||||||
});
|
|
||||||
instanceId = raised.instanceId;
|
|
||||||
held = raised.images;
|
|
||||||
|
|
||||||
await must(`cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
||||||
await must(`${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
|
||||||
const up = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
||||||
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await mesh(`node add ${MACHINE}`);
|
|
||||||
const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
|
|
||||||
await must(`${HOST_PATH} enrol --token ${quote(token)}`);
|
|
||||||
await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
||||||
}, { timeout: 1_800_000 });
|
|
||||||
|
|
||||||
after(async () => {
|
|
||||||
if (instanceId) await destroy(instanceId);
|
|
||||||
await destroyAll(`${SCENARIO}-`);
|
|
||||||
}, { timeout: 600_000 });
|
|
||||||
|
|
||||||
test("redis creates a consumer's login with the password the mesh minted, sealing nothing", {
|
|
||||||
skip, timeout: 900_000,
|
|
||||||
}, async () => {
|
|
||||||
// redis as a provider: the server, and a broker-bound runtime that serves its tools AND runs its
|
|
||||||
// provisioner. The provisioner is pointed at the contributions file the mesh would write
|
|
||||||
// (MESH_RECEIVES). There is NO MESH_SEAL_KEY — the whole point of ADR 0048 is that a provider
|
|
||||||
// needs none.
|
|
||||||
const manifest = JSON.stringify({
|
|
||||||
module: "redis",
|
|
||||||
version: "1",
|
|
||||||
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
|
|
||||||
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
|
|
||||||
resources: [
|
|
||||||
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
|
|
||||||
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
|
|
||||||
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
|
|
||||||
{
|
|
||||||
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
|
|
||||||
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/services/redis/data:/data",
|
|
||||||
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
|
|
||||||
],
|
|
||||||
args: ["/etc/redis/redis.conf"],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
|
|
||||||
network: "host",
|
|
||||||
volumes: [
|
|
||||||
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
|
|
||||||
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
|
|
||||||
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
|
|
||||||
],
|
|
||||||
env: {
|
|
||||||
MESH_BROKER_FILE: "/run/secrets/broker",
|
|
||||||
MESH_RECEIVES: "/var/lib/redis-module/grants/redis-cache.json",
|
|
||||||
MESH_PROVISION_REDIS: "127.0.0.1:6379",
|
|
||||||
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/redis.json && docker cp /tmp/redis.json mesh-controller:/redis.json`);
|
|
||||||
await mesh("module add /redis.json");
|
|
||||||
await mesh(`module issue redis --node ${MACHINE}`);
|
|
||||||
await mesh(`assign ${MACHINE} redis`);
|
|
||||||
await mesh(`push ${MACHINE}`);
|
|
||||||
await settled();
|
|
||||||
|
|
||||||
const running = await must(`docker ps --format '{{.Names}}'`);
|
|
||||||
assert.match(running, /mesh-redis/, `redis's runtime is not running:\n${(await on(`docker logs mesh-redis 2>&1 | tail -20`)).out}`);
|
|
||||||
|
|
||||||
// What the mesh delivers to the provider: a contributions file naming the consumer's login and
|
|
||||||
// where its password is, and the password itself as the file the host leaves after unsealing.
|
|
||||||
const password = "mesh-minted-9f3c2a";
|
|
||||||
await must(`printf %s ${quote(password)} > /var/lib/redis-module/grants/app.secret`);
|
|
||||||
const contributions = JSON.stringify({
|
|
||||||
contributions: 1,
|
|
||||||
requirement: "redis-cache",
|
|
||||||
generated: "by the mesh — do not edit",
|
|
||||||
given: [
|
|
||||||
{ from: "app", node: "app-node", at: "192.0.2.20:6379", as: "app-one", secret: "/var/lib/redis-module/grants/app.secret", values: {} },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
await must(`printf %s ${quote(contributions)} > /var/lib/redis-module/grants/redis-cache.json`);
|
|
||||||
|
|
||||||
// Within a reconcile tick the provisioner creates the ACL user. It exists on the server.
|
|
||||||
let acl = "";
|
|
||||||
const until = Date.now() + 60_000;
|
|
||||||
while (Date.now() < until) {
|
|
||||||
acl = (await on(`docker exec redis redis-cli -a ${quote(await must(`cat /var/lib/redis-module/default.secret`))} --no-auth-warning ACL LIST 2>/dev/null`)).out;
|
|
||||||
if (/app-one/.test(acl)) break;
|
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
|
||||||
}
|
|
||||||
assert.match(acl, /app-one/, `the provisioner never created the consumer's login:\n${(await on(`docker logs mesh-redis 2>&1 | tail -30`)).out}\n---\n${acl}`);
|
|
||||||
|
|
||||||
// The proof: authenticate as that consumer with the password the MESH minted. PONG means the
|
|
||||||
// provisioner created the login with exactly that password. A provisioner that invented its own
|
|
||||||
// (the old behaviour) would answer WRONGPASS here.
|
|
||||||
const authed = await on(`docker exec redis redis-cli --user app-one --pass ${quote(password)} --no-auth-warning PING 2>&1`);
|
|
||||||
assert.doesNotMatch(authed.out, /WRONGPASS/,
|
|
||||||
`the consumer could not authenticate with the mesh's password — the provider used a different one:\n${authed.out}`);
|
|
||||||
assert.match(authed.out, /PONG/, `expected PONG authenticating as the consumer:\n${authed.out}`);
|
|
||||||
|
|
||||||
// And it needed no seal key: the runtime came up and provisioned with MESH_SEAL_KEY set nowhere.
|
|
||||||
const env = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`);
|
|
||||||
assert.doesNotMatch(env, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${env}`);
|
|
||||||
|
|
||||||
// The provisioner emitted its lifecycle event under the bound account, and no emit was refused.
|
|
||||||
const log = (await on(`docker logs mesh-redis 2>&1`)).out;
|
|
||||||
assert.doesNotMatch(log, /emit .*failed/, `the provisioned event was refused:\n${log}`);
|
|
||||||
});
|
|
||||||
@@ -37,7 +37,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, catalogueModule, catalogueIsPresent, deriveTheFilterOn } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -50,12 +50,14 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "route-forwarding";
|
const SCENARIO = "route-forwarding";
|
||||||
const MACHINE = "anchor";
|
const MACHINE = "anchor";
|
||||||
const NAME = "hello.example";
|
const NAME = "hello.example";
|
||||||
const PAGE = "hello from hello-web, routed by the mesh";
|
const PAGE = "hello from hello-web, routed by the mesh";
|
||||||
|
/** The node's public domain; hello-web's label composes under it (ADR 0066). */
|
||||||
|
const DOMAIN = "example";
|
||||||
|
|
||||||
let instanceId = "";
|
let instanceId = "";
|
||||||
let held: HeldImage[] = [];
|
let held: HeldImage[] = [];
|
||||||
@@ -85,10 +87,6 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** The reference a manifest should carry, once this scenario has been raised. */
|
/** The reference a manifest should carry, once this scenario has been raised. */
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
|
/** The foundation bundle: ours by the ID the machine holds, everything else upstream. */
|
||||||
function bundleFor(images: HeldImage[]): string {
|
function bundleFor(images: HeldImage[]): string {
|
||||||
@@ -167,64 +165,31 @@ after(async () => {
|
|||||||
test("the mesh routes a public name through the proxy to the consumer, and withdraws it on unassign", {
|
test("the mesh routes a public name through the proxy to the consumer, and withdraws it on unassign", {
|
||||||
skip, timeout: 1_500_000,
|
skip, timeout: 1_500_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
// The PROVIDER: route-proxy in the plain-HTTP shape — provides `route`, is given every consumer as
|
// All three from the catalogue (novox/hq ADR 0093, issue 074): the authority the proxy requires,
|
||||||
// the file at receives.route, forwards by Host. No TLS here (that is certificates.test.ts); the
|
// the proxy, and the consumer. The proxy's manifest names the runtime image the scenario stocks;
|
||||||
// image is pinned to what this scenario serves by digest.
|
// the authority and the consumer's server are pulled upstream by digest. The name the consumer
|
||||||
const proxyManifest = JSON.stringify({
|
// is routed under is composed from its label and the node's public domain (ADR 0066), which
|
||||||
module: "route-proxy",
|
// the bed sets first.
|
||||||
version: "1",
|
await mesh(`node public-domain ${MACHINE} ${DOMAIN}`);
|
||||||
capabilities: ["container-runtime"],
|
// The authority certifies itself for the machine's private-network address, which is what a
|
||||||
provides: [{ name: "route", scope: "mesh" }],
|
// consumer on any node dials (ADR 0098); a machine raised from the bundle has none until it is
|
||||||
serves: { route: {} },
|
// placed on the overlay. Placed as a hub of one, the way a real first node is, and converged
|
||||||
receives: { route: "/var/lib/route-proxy/routes/mesh.json" },
|
// BEFORE the modules arrive: the proxy fetches the authority's roots at that address at first
|
||||||
listens: [{ port: 80, protocol: "tcp", from: "anywhere", why: "public HTTP; the route-forwarding front door" }],
|
// start, so the interface must exist by then — in one push the order between modules is not
|
||||||
resources: [
|
// promised. The derived filter admits the hub's port, as genesis does on a control-node (ADR 0088).
|
||||||
{ id: "state", type: "directory", path: "/var/lib/route-proxy", mode: "0700" },
|
await mesh(`overlay place ${MACHINE} --hub --endpoint 192.0.2.10:51820 --site lab`);
|
||||||
{ id: "routes-dir", type: "directory", path: "/var/lib/route-proxy/routes", mode: "0700" },
|
await mesh(`assign ${MACHINE} networking`);
|
||||||
{
|
await mesh(`push ${MACHINE}`);
|
||||||
id: "server", type: "container", name: "route-proxy",
|
await settled();
|
||||||
image: pinned("mesh-route-proxy"), network: "host",
|
await deriveTheFilterOn({ machine: MACHINE, node: MACHINE, hubPort: 51820,
|
||||||
volumes: ["/var/lib/route-proxy/routes:/routes:ro"],
|
must: (_m, c, t) => must(c, t), mesh, on: (_m, c, t) => on(c, t) });
|
||||||
env: { ROUTES: "/routes/mesh.json", LISTEN: ":80" },
|
const overlay = await must(`ip -4 addr show dev mesh0 2>&1 || ip -4 addr 2>&1`);
|
||||||
},
|
assert.match(overlay, /inet 10\./, `${MACHINE} has no private-network address after networking converged:\n${overlay}`);
|
||||||
],
|
for (const name of ["step-ca", "route-proxy", "hello-web"]) {
|
||||||
});
|
await must(`printf %s ${quote(catalogueModule(name, held))} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
||||||
|
await mesh(`module add /${name}.json`);
|
||||||
// The CONSUMER: hello-web requires `route` and contributes the name it wants and the port it
|
await mesh(`assign ${MACHINE} ${name}`);
|
||||||
// listens on. It runs no code of the mesh's — a bare alpine serving a fixed page over a busybox nc
|
}
|
||||||
// loop stands in for a web service. `contributes` is what makes it *ask*: the grant forms from it.
|
|
||||||
const webManifest = JSON.stringify({
|
|
||||||
module: "hello-web",
|
|
||||||
slug: "hello",
|
|
||||||
version: "1",
|
|
||||||
capabilities: ["container-runtime"],
|
|
||||||
requires: ["route"],
|
|
||||||
contributes: { route: { name: NAME, port: 8080 } },
|
|
||||||
binds: { route: "/var/lib/hello-web/route.json" },
|
|
||||||
listens: [{ port: 8080, protocol: "tcp", from: "mesh", why: "the demo page; only the proxy reaches it" }],
|
|
||||||
resources: [
|
|
||||||
{ id: "state", type: "directory", path: "/var/lib/hello-web", mode: "0700" },
|
|
||||||
{ id: "page", type: "file", path: "/var/lib/hello-web/index.html", mode: "0644", content: `${PAGE}\n` },
|
|
||||||
{ id: "net", type: "network", name: "hello-web" },
|
|
||||||
{
|
|
||||||
id: "server", type: "container", name: "hello-web",
|
|
||||||
image: pinned("alpine"), network: "hello-web", ports: ["8080:8080"],
|
|
||||||
volumes: ["/var/lib/hello-web/index.html:/www/index.html:ro"],
|
|
||||||
args: ["sh", "-c",
|
|
||||||
"while true; do { printf 'HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nConnection: close\\r\\n\\r\\n'; cat /www/index.html; } | nc -l -p 8080; done"],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(proxyManifest)} > /tmp/route-proxy.json && docker cp /tmp/route-proxy.json mesh-controller:/route-proxy.json`);
|
|
||||||
await mesh("module add /route-proxy.json");
|
|
||||||
// No `module issue`: route-proxy has no broker account and no own-secret to mint. `assign` resolves
|
|
||||||
// its plan and the provider is matchable by a consumer's route from that alone.
|
|
||||||
await mesh(`assign ${MACHINE} route-proxy`);
|
|
||||||
|
|
||||||
await must(`printf %s ${quote(webManifest)} > /tmp/hello-web.json && docker cp /tmp/hello-web.json mesh-controller:/hello-web.json`);
|
|
||||||
await mesh("module add /hello-web.json");
|
|
||||||
await mesh(`assign ${MACHINE} hello-web`);
|
|
||||||
|
|
||||||
await mesh(`push ${MACHINE}`);
|
await mesh(`push ${MACHINE}`);
|
||||||
await settled();
|
await settled();
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
* service has: the runtime names its config resource, and the host recreates the container when that
|
* service has: the runtime names its config resource, and the host recreates the container when that
|
||||||
* resource changed this pass.
|
* resource changed this pass.
|
||||||
*
|
*
|
||||||
* This assigns grafana configured by settings, then changes the token and pushes again, and asserts
|
* This assigns a settings-configured runtime (the fixture wears no catalogue name; its image is grafana's tool runtime), then changes the token and pushes again, and asserts
|
||||||
* the container was replaced (a new container id) and the config on disk carries the new value.
|
* the container was replaced (a new container id) and the config on disk carries the new value.
|
||||||
* It builds the host from source (no --no-build), because the behaviour under test is the host's.
|
* It builds the host from source (no --no-build), because the behaviour under test is the host's.
|
||||||
*
|
*
|
||||||
@@ -113,7 +113,7 @@ async function settled(withinMs = 480_000): Promise<void> {
|
|||||||
async function setToken(token: string): Promise<void> {
|
async function setToken(token: string): Promise<void> {
|
||||||
const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token });
|
const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token });
|
||||||
await must(`printf %s ${quote(settings)} > /tmp/s.json && docker cp /tmp/s.json mesh-controller:/s.json`);
|
await must(`printf %s ${quote(settings)} > /tmp/s.json && docker cp /tmp/s.json mesh-controller:/s.json`);
|
||||||
await mesh(`settings set grafana /s.json --node ${MACHINE}`);
|
await mesh(`settings set a-runtime /s.json --node ${MACHINE}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function containerId(): Promise<string> {
|
async function containerId(): Promise<string> {
|
||||||
@@ -151,7 +151,7 @@ test("a running runtime is recreated when its settings change, and reads the new
|
|||||||
skip, timeout: 900_000,
|
skip, timeout: 900_000,
|
||||||
}, async () => {
|
}, async () => {
|
||||||
const manifest = JSON.stringify({
|
const manifest = JSON.stringify({
|
||||||
module: "grafana",
|
module: "a-runtime",
|
||||||
version: "1",
|
version: "1",
|
||||||
emits: ["module.grafana.alert.firing"],
|
emits: ["module.grafana.alert.firing"],
|
||||||
"own-secrets": { broker: "/var/lib/mesh/grafana/broker" },
|
"own-secrets": { broker: "/var/lib/mesh/grafana/broker" },
|
||||||
@@ -170,12 +170,12 @@ test("a running runtime is recreated when its settings change, and reads the new
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
await must(`printf %s ${quote(manifest)} > /tmp/grafana.json && docker cp /tmp/grafana.json mesh-controller:/grafana.json`);
|
await must(`printf %s ${quote(manifest)} > /tmp/a-runtime.json && docker cp /tmp/a-runtime.json mesh-controller:/a-runtime.json`);
|
||||||
await mesh("module add /grafana.json");
|
await mesh("module add /a-runtime.json");
|
||||||
|
|
||||||
await setToken("token-alpha");
|
await setToken("token-alpha");
|
||||||
await mesh(`module issue grafana --node ${MACHINE}`);
|
await mesh(`module issue a-runtime --node ${MACHINE}`);
|
||||||
await mesh(`assign ${MACHINE} grafana`);
|
await mesh(`assign ${MACHINE} a-runtime`);
|
||||||
await mesh(`push ${MACHINE}`);
|
await mesh(`push ${MACHINE}`);
|
||||||
await settled();
|
await settled();
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
/**
|
||||||
|
* **Nothing a machine sends while the store restarts is lost** (novox/hq issues 082, 083).
|
||||||
|
*
|
||||||
|
* The foundation's store is recreated when it is adopted, and for those seconds the control plane
|
||||||
|
* cannot write. This bed takes the store away on the control-node, has a second machine enrol into
|
||||||
|
* the gap, and brings the store back after the control plane has had to say "not now":
|
||||||
|
*
|
||||||
|
* - a report arriving in the gap is held, and recorded when the store is back;
|
||||||
|
* - the enrolment is answered "not now" while that report is held — not queued behind it — and
|
||||||
|
* completes on its own when the store is back, with the keys it started with: the mesh holds
|
||||||
|
* the very keys the machine generated;
|
||||||
|
* - the machine then applies what it is pushed and is heard from.
|
||||||
|
*
|
||||||
|
* It needs a host binary and the foundation bundle:
|
||||||
|
*
|
||||||
|
* MESH_LAB_HOST_BINARY=.../mesh-host
|
||||||
|
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
||||||
|
*/
|
||||||
|
import { test, before, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync } from "node:fs";
|
||||||
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
||||||
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
|
const capability = await labIsUsable();
|
||||||
|
const binary = hostBinaryPath();
|
||||||
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
|
|
||||||
|
const skip = !capability.usable
|
||||||
|
? `lab not usable: ${capability.why}`
|
||||||
|
: !binary || !existsSync(binary)
|
||||||
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
|
: !bundle || !existsSync(bundle)
|
||||||
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
|
: false;
|
||||||
|
|
||||||
|
const SCENARIO = "store-window";
|
||||||
|
let instanceId = "";
|
||||||
|
let held: HeldImage[] = [];
|
||||||
|
|
||||||
|
function quote(s: string): string {
|
||||||
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
||||||
|
const { stdout } = await exec(instanceId, machine, [
|
||||||
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
||||||
|
], timeoutMs);
|
||||||
|
const marker = stdout.lastIndexOf("__exit=");
|
||||||
|
if (marker < 0) return { out: stdout, ok: false };
|
||||||
|
return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
const { out, ok } = await on(machine, command, timeoutMs);
|
||||||
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
||||||
|
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
function tokenFrom(said: string): string {
|
||||||
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
||||||
|
assert.ok(found, `no token in:\n${said}`);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function until(what: string, within: number, check: () => Promise<boolean>, why: () => Promise<string>): Promise<void> {
|
||||||
|
const end = Date.now() + within;
|
||||||
|
while (Date.now() < end) {
|
||||||
|
if (await check()) return;
|
||||||
|
await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
}
|
||||||
|
assert.fail(`${what} did not happen within ${Math.round(within / 1000)}s:\n${await why()}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
if (skip) return;
|
||||||
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
|
});
|
||||||
|
instanceId = raised.instanceId;
|
||||||
|
held = raised.images;
|
||||||
|
|
||||||
|
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${foundationBundle(bundle, raised.images)}\nMESHBUNDLE`);
|
||||||
|
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`, 600_000);
|
||||||
|
const up = await must("anchor", `docker ps --format '{{.Names}}'`);
|
||||||
|
for (const c of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
||||||
|
assert.match(up, new RegExp(c), `the foundation did not raise ${c}:\n${up}`);
|
||||||
|
}
|
||||||
|
await mesh("node add anchor");
|
||||||
|
const own = tokenFrom(await mesh("token issue --node anchor"));
|
||||||
|
await must("anchor", `${HOST_PATH} enrol --token ${quote(own)}`);
|
||||||
|
await must("anchor", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||||
|
}, { timeout: 1_800_000 });
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
if (process.env["MESH_LAB_KEEP"]) { console.log(`MESH_LAB_KEEP set — leaving ${instanceId} standing`); return; }
|
||||||
|
if (instanceId) await destroy(instanceId);
|
||||||
|
await destroyAll(`${SCENARIO}-`);
|
||||||
|
}, { timeout: 600_000 });
|
||||||
|
|
||||||
|
test("a machine enrolling while the store is away joins when it comes back, with the keys it started with", {
|
||||||
|
skip, timeout: 900_000,
|
||||||
|
}, async () => {
|
||||||
|
await mesh("node add laptop");
|
||||||
|
const token = tokenFrom(await mesh("token issue --node laptop"));
|
||||||
|
|
||||||
|
// A report that will arrive in the gap: the anchor is pushed a step that takes ten seconds, so
|
||||||
|
// its report lands after the store has gone.
|
||||||
|
const sleeper = onTheMachine("alpine", held);
|
||||||
|
await must("anchor", `printf %s '{"module":"slow","version":"1","resources":[` +
|
||||||
|
`{"id":"step","type":"container","name":"slow-step","image":"${sleeper}","run-once":true,` +
|
||||||
|
`"args":["sh","-c","sleep 10"]}]}' > /tmp/slow.json`);
|
||||||
|
await must("anchor", `docker cp /tmp/slow.json mesh-controller:/slow.json`);
|
||||||
|
await mesh("module add /slow.json");
|
||||||
|
await mesh("assign anchor slow");
|
||||||
|
await mesh("push anchor");
|
||||||
|
|
||||||
|
// The store goes away, as it does when the foundation is adopted; the broker stays, so the
|
||||||
|
// report and the enrolment reach the control plane and the control plane cannot write.
|
||||||
|
await must("anchor", `docker stop mesh-store`);
|
||||||
|
await until("the anchor's report arriving in the gap and being held", 120_000,
|
||||||
|
async () => /could not keep anchor's report .*holding it/.test((await on("anchor", `docker logs mesh-controller 2>&1`)).out),
|
||||||
|
async () => `--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}\n` +
|
||||||
|
`--- anchor host ---\n${(await on("anchor", `tail -10 /var/log/mesh-host.log`)).out}`);
|
||||||
|
|
||||||
|
// The machine enrols into the gap. In the background: it will be told "not now" and keep asking.
|
||||||
|
await must("laptop", `nohup sh -c ${quote(`${HOST_PATH} enrol --token ${quote(token)} > /tmp/enrol.log 2>&1; ` +
|
||||||
|
`echo "exit=$?" >> /tmp/enrol.log`)} > /dev/null 2>&1 & sleep 1`);
|
||||||
|
|
||||||
|
// The control plane said "not now" at least once, while the anchor's report was held — so the
|
||||||
|
// gap was hit, and the enrolment was answered rather than queued behind what the store owed.
|
||||||
|
await until("the control plane asking the machine to enrol again", 90_000,
|
||||||
|
async () => /asked "laptop" to enrol again/.test((await on("anchor", `docker logs mesh-controller 2>&1`)).out),
|
||||||
|
async () => `--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}\n` +
|
||||||
|
`--- laptop enrol ---\n${(await on("laptop", `cat /tmp/enrol.log`)).out}`);
|
||||||
|
|
||||||
|
// The store comes back; the enrolment completes on its own, with nothing done by hand.
|
||||||
|
await must("anchor", `docker start mesh-store`);
|
||||||
|
await until("the enrolment completing", 150_000,
|
||||||
|
async () => /exit=/.test((await on("laptop", `cat /tmp/enrol.log`)).out),
|
||||||
|
async () => `--- laptop enrol ---\n${(await on("laptop", `cat /tmp/enrol.log`)).out}\n` +
|
||||||
|
`--- controller ---\n${(await on("anchor", `docker logs --tail 30 mesh-controller 2>&1`)).out}`);
|
||||||
|
const said = (await on("laptop", `cat /tmp/enrol.log`)).out;
|
||||||
|
assert.match(said, /exit=0/, `the enrolment did not complete after the store came back:\n${said}`);
|
||||||
|
assert.match(said, /enrolled as laptop/, `the machine was not enrolled as laptop:\n${said}`);
|
||||||
|
|
||||||
|
// The mesh holds the very keys the machine generated at the start: its identity is the live key,
|
||||||
|
// and its sealing key is the one on its record.
|
||||||
|
const identity = said.match(/generated this node's identity: (\S+)/)?.[1];
|
||||||
|
const sealing = said.match(/generated this node's sealing key:\s+(\S+)/)?.[1];
|
||||||
|
assert.ok(identity && sealing, `the enrolment did not say which keys it generated:\n${said}`);
|
||||||
|
const nodeId = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||||
|
`-c "select id from node where name = 'laptop'"`)).trim();
|
||||||
|
const live = (await must("anchor", `docker exec mesh-store psql -U postgres -d identity -qAt ` +
|
||||||
|
`-c "select encode(public, 'base64') from node_key where node = '${nodeId}' and revoked is null"`)).trim();
|
||||||
|
assert.equal(live, identity, `the mesh's live key for laptop is not the one it generated`);
|
||||||
|
const sealedTo = (await must("anchor", `docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
||||||
|
`-c "select sealing_key from node where name = 'laptop'"`)).trim();
|
||||||
|
assert.equal(sealedTo, sealing, `the mesh's sealing key for laptop is not the one it generated`);
|
||||||
|
|
||||||
|
// The report held through the gap was recorded once the store was back.
|
||||||
|
await until("the anchor's held report being recorded", 60_000,
|
||||||
|
async () => {
|
||||||
|
const r = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
|
||||||
|
if (!r.ok) return false;
|
||||||
|
const state = JSON.parse(r.out) as { reported: { node: string; outcome: string; current: boolean }[] };
|
||||||
|
return state.reported.some((w) => w.node === "anchor" && w.outcome === "applied" && w.current);
|
||||||
|
},
|
||||||
|
async () => (await on("anchor", `docker logs --tail 20 mesh-controller 2>&1`)).out);
|
||||||
|
|
||||||
|
// And the machine is a working member: pushed something, it applies it and is heard from.
|
||||||
|
await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
||||||
|
await must("anchor", `printf %s '{"module":"marker","version":"1","resources":[` +
|
||||||
|
`{"id":"marker","type":"file","path":"/etc/store-window","content":"joined\\\\n","mode":"0644"}]}' > /tmp/marker.json`);
|
||||||
|
await must("anchor", `docker cp /tmp/marker.json mesh-controller:/marker.json`);
|
||||||
|
await mesh("module add /marker.json");
|
||||||
|
await mesh("assign laptop marker");
|
||||||
|
await mesh("push laptop");
|
||||||
|
await until("the machine applying what it was pushed", 120_000,
|
||||||
|
async () => (await on("laptop", `grep -q joined /etc/store-window`)).ok,
|
||||||
|
async () => (await on("laptop", `tail -20 /var/log/mesh-host.log`)).out);
|
||||||
|
await until("the mesh hearing the machine's report", 120_000,
|
||||||
|
async () => {
|
||||||
|
const r = await on("anchor", `docker exec mesh-controller /mesh-controller status --json`);
|
||||||
|
if (!r.ok) return false;
|
||||||
|
const state = JSON.parse(r.out) as { reported: { node: string; outcome: string; current: boolean }[] };
|
||||||
|
return state.reported.some((w) => w.node === "laptop" && w.outcome === "applied" && w.current);
|
||||||
|
},
|
||||||
|
async () => (await on("anchor", `docker exec mesh-controller /mesh-controller status 2>&1`)).out);
|
||||||
|
});
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
* whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set.
|
* whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set.
|
||||||
*
|
*
|
||||||
* Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the
|
* Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the
|
||||||
* `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
|
* `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres
|
||||||
* providers co-located with them. The media stack shares the operator-owned library directories
|
* providers co-located with them. The media stack shares the operator-owned library directories
|
||||||
* under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS
|
* under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS
|
||||||
* each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those
|
* each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those
|
||||||
@@ -25,19 +25,17 @@
|
|||||||
|
|
||||||
import { test, before, after } from "node:test";
|
import { test, before, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { dirname, resolve } from "node:path";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, deriveTheFilterOn, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
const binary = hostBinaryPath();
|
const binary = hostBinaryPath();
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
const skip = !capability.usable
|
||||||
? `lab not usable: ${capability.why}`
|
? `lab not usable: ${capability.why}`
|
||||||
@@ -45,14 +43,12 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "whole-mesh-ace";
|
const SCENARIO = "whole-mesh-ace";
|
||||||
const NODE = "ace";
|
const NODE = "ace";
|
||||||
|
|
||||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
|
||||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
|
||||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
|
||||||
|
|
||||||
/** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */
|
/** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */
|
||||||
const MEDIA_DIRS = [
|
const MEDIA_DIRS = [
|
||||||
@@ -175,27 +171,17 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
function bundleFor(images: HeldImage[]): string {
|
||||||
return foundationBundle(bundle, images);
|
return foundationBundle(bundle, images);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The catalogue's manifest as the lab runs it (harness). This bed's own loader never resolved a
|
||||||
|
* runtime ARTIFACT to a stocked image, so every module whose runtime the mesh builds travelled to
|
||||||
|
* the machine unresolved — the shared loader does (novox/hq 04-ISSUES/073). */
|
||||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||||
const path = resolve(catalogDir, name, "module.json");
|
const manifest = catalogueModule(name, held, { ports: REMAP[name] });
|
||||||
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
return { manifest, broker: needsBrokerAccount(manifest) };
|
||||||
resources?: { type: string; image?: string; ports?: string[] }[];
|
|
||||||
};
|
|
||||||
const remap = REMAP[name] ?? {};
|
|
||||||
for (const r of m.resources ?? []) {
|
|
||||||
if (r.type !== "container") continue;
|
|
||||||
if (typeof r.image === "string") r.image = pinned(r.image);
|
|
||||||
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
|
|
||||||
}
|
|
||||||
const manifest = JSON.stringify(m);
|
|
||||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
function tokenFrom(said: string): string {
|
||||||
@@ -240,7 +226,6 @@ async function nodeState(node: string): Promise<NodeState> {
|
|||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
if (skip) return;
|
if (skip) return;
|
||||||
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
@@ -279,7 +264,7 @@ test("the whole ace service set resolves, installs and converges on one node in
|
|||||||
}, async () => {
|
}, async () => {
|
||||||
for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
|
for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
|
||||||
|
|
||||||
// The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres/redis).
|
// The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres).
|
||||||
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
||||||
await mesh(`overlay place ${NODE} --site lab`);
|
await mesh(`overlay place ${NODE} --site lab`);
|
||||||
await mesh("assign anchor networking");
|
await mesh("assign anchor networking");
|
||||||
|
|||||||
@@ -58,21 +58,20 @@ import { test, before, after } from "node:test";
|
|||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { dirname, join, resolve } from "node:path";
|
import { join, resolve } from "node:path";
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts";
|
||||||
import {
|
import {
|
||||||
bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH,
|
bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH,
|
||||||
} from "../../src/lifecycle/place.ts";
|
} from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, catalogueDir, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts";
|
||||||
import { referenceFor, type HeldImage } from "../../src/pinning.ts";
|
import { referenceFor, type HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
const binary = hostBinaryPath();
|
const binary = hostBinaryPath();
|
||||||
const installer = bootstrapBinaryPath();
|
const installer = bootstrapBinaryPath();
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
|
||||||
// What the installer is told to build. It carries a builder rather than a finished control plane
|
// What the installer is told to build. It carries a builder rather than a finished control plane
|
||||||
// (novox/hq ADR 0073), so genesis is given a repository and a commit — and a commit rather than a
|
// (novox/hq ADR 0073), so genesis is given a repository and a commit — and a commit rather than a
|
||||||
// branch, because what is cloned is the trust anchor for everything this mesh will ever run.
|
// branch, because what is cloned is the trust anchor for everything this mesh will ever run.
|
||||||
@@ -93,7 +92,7 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_SOURCE is not set to the repository the control plane is built from"
|
? "MESH_LAB_SOURCE is not set to the repository the control plane is built from"
|
||||||
: !sourceRef
|
: !sourceRef
|
||||||
? "MESH_LAB_SOURCE_REF is not set to the commit to build"
|
? "MESH_LAB_SOURCE_REF is not set to the commit to build"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "whole-mesh-full";
|
const SCENARIO = "whole-mesh-full";
|
||||||
/** novox hosts the foundation and the control plane; it is where `mesh` commands run. */
|
/** novox hosts the foundation and the control plane; it is where `mesh` commands run. */
|
||||||
@@ -154,9 +153,8 @@ const PUBLIC_DOMAIN: Record<string, string> = { novox: "novox.incus", ace: "zura
|
|||||||
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
const KEEP = !!process.env["MESH_LAB_KEEP"];
|
||||||
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined);
|
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined);
|
||||||
|
|
||||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
/** The catalogue's modules directory (harness). Absent, the bed skips — see `skip`. */
|
||||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
const catalogDir = catalogueIsPresent() ? "" : catalogueDir();
|
||||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
|
||||||
|
|
||||||
const MEDIA_DIRS = [
|
const MEDIA_DIRS = [
|
||||||
"/services/media/series", "/services/media/anime", "/services/media/movies",
|
"/services/media/series", "/services/media/anime", "/services/media/movies",
|
||||||
@@ -181,6 +179,9 @@ const NOVOX: Mod[] = [
|
|||||||
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
|
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
|
||||||
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
|
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
|
||||||
{ name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] },
|
{ name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] },
|
||||||
|
// The vault, before everything that keeps a secret from it: gitea, umami, influxdb, mailu
|
||||||
|
// require one (novox/hq ADRs 0085, 0094).
|
||||||
|
{ name: "mesh-vault", containers: ["mesh-vault"] },
|
||||||
// ADR 0066: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or
|
// ADR 0066: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or
|
||||||
// route-proxy is unresolvable and takes every routed module down with it. step-ca is that
|
// route-proxy is unresolvable and takes every routed module down with it. step-ca is that
|
||||||
// provider, on the anchor, at mesh scope.
|
// provider, on the anchor, at mesh scope.
|
||||||
@@ -203,7 +204,7 @@ const NOVOX: Mod[] = [
|
|||||||
// what an operator's `module add` + `assign` would do on a mesh that already has it, and a
|
// what an operator's `module add` + `assign` would do on a mesh that already has it, and a
|
||||||
// module the installer put there had better survive being asked for a second time. It also keeps
|
// module the installer put there had better survive being asked for a second time. It also keeps
|
||||||
// mesh-registry in the convergence report, where a reader expects to see it.
|
// mesh-registry in the convergence report, where a reader expects to see it.
|
||||||
{ name: "registry", containers: ["mesh-registry"] },
|
{ name: "distribution", containers: ["mesh-registry"] },
|
||||||
{
|
{
|
||||||
name: "mailu",
|
name: "mailu",
|
||||||
containers: [
|
containers: [
|
||||||
@@ -212,16 +213,16 @@ const NOVOX: Mod[] = [
|
|||||||
"mailu-front", "mesh-mailu",
|
"mailu-front", "mesh-mailu",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{ name: "firewall", containers: [], node: true },
|
{ name: "nftables", containers: [], node: true },
|
||||||
{ name: "fail2ban", containers: [], node: true },
|
{ name: "fail2ban", containers: [], node: true },
|
||||||
];
|
];
|
||||||
const CORE_NOVOX = new Set([
|
const CORE_NOVOX = new Set([
|
||||||
"postgres", "redis", "minio", "mongodb", "mssql", "lavinmq",
|
"postgres", "redis", "minio", "mongodb", "mssql", "lavinmq",
|
||||||
"route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be",
|
"route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be",
|
||||||
"portainer", "verdaccio", "registry",
|
"portainer", "verdaccio", "distribution",
|
||||||
]);
|
]);
|
||||||
const GAPS_NOVOX = new Set([
|
const GAPS_NOVOX = new Set([
|
||||||
"umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder",
|
"umami", "mailu", "nftables", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder",
|
||||||
// step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in
|
// step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in
|
||||||
// mesh-controller, and this bed is not the place to discover that a fix has not landed yet. What is
|
// mesh-controller, and this bed is not the place to discover that a fix has not landed yet. What is
|
||||||
// gated is the half that is decided and cheap — see the ADR 0066 section at the end.
|
// gated is the half that is decided and cheap — see the ADR 0066 section at the end.
|
||||||
@@ -308,18 +309,15 @@ const CREDENTIALS: { node: string; module: string; name: string; crash: string }
|
|||||||
{ node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" },
|
{ node: "ace", module: "home-assistant", name: "token", crash: "no Home Assistant token" },
|
||||||
{ node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" },
|
{ node: "ace", module: "nzbget", name: "password", crash: "NZBGet not configured" },
|
||||||
{ node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" },
|
{ node: "ace", module: "qbittorrent", name: "password", crash: "qBittorrent not configured" },
|
||||||
{ node: "novox", module: "umami", name: "admin", crash: "admin password is not set" },
|
|
||||||
];
|
];
|
||||||
|
|
||||||
/** Operator secrets for the credential modules that own-secret their whole app (mailu, de-spiegel). */
|
/** Operator secrets for the credential modules that own-secret their whole app (de-spiegel,
|
||||||
|
* amqp-email-forwarder). mailu's and umami's are kept in the vault now (ADR 0094), not delivered. */
|
||||||
const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [
|
const OPERATOR_SECRETS: { node: string; module: string; name: string; value: string }[] = [
|
||||||
{ node: "novox", module: "mailu", name: "secret-key", value: "0123456789abcdef0123456789abcdef" },
|
|
||||||
{ node: "novox", module: "mailu", name: "admin", value: "MailuAdminFakePass123" },
|
|
||||||
{ node: "novox", module: "mailu", name: "api-token", value: "mailuapitokenfake0123456789abcd" },
|
|
||||||
{ node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" },
|
{ node: "novox", module: "de-spiegel", name: "smtp-user", value: "despiegel-smtp-fake" },
|
||||||
{ node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" },
|
{ node: "novox", module: "de-spiegel", name: "smtp-pass", value: "despiegel-pass-fake" },
|
||||||
{ node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" },
|
{ node: "novox", module: "amqp-email-forwarder", name: "smtp-user", value: "eef-smtp-fake" },
|
||||||
{ node: "novox", module: "amqp-email-forwarder", name: "smtp-pass", value: "eef-pass-fake" },
|
{ node: "novox", module: "amqp-email-forwarder", name: "smtp-password", value: "eef-pass-fake" },
|
||||||
];
|
];
|
||||||
|
|
||||||
let instanceId = "";
|
let instanceId = "";
|
||||||
@@ -350,45 +348,16 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
||||||
function pinned(reference: string): string {
|
|
||||||
return onTheMachine(reference, held);
|
|
||||||
}
|
|
||||||
|
|
||||||
function bundleFor(images: HeldImage[]): string {
|
function bundleFor(images: HeldImage[]): string {
|
||||||
return foundationBundle(bundle, images);
|
return foundationBundle(bundle, images);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The catalogue's manifest as the lab runs it (harness): artifacts resolved to the images the
|
||||||
|
* scenario stocked — the lab standing in for the builder — images pinned, host ports remapped. */
|
||||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||||
const path = resolve(catalogDir, name, "module.json");
|
const manifest = catalogueModule(name, held, { ports: REMAP[name] });
|
||||||
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
return { manifest, broker: needsBrokerAccount(manifest) };
|
||||||
resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[];
|
|
||||||
};
|
|
||||||
const remap = REMAP[name] ?? {};
|
|
||||||
for (const r of m.resources ?? []) {
|
|
||||||
if (r.type !== "container") continue;
|
|
||||||
// **A container naming an artifact is a module the mesh builds, and this bed does not build.**
|
|
||||||
// It pre-builds the same images on the workstation and stocks them, which is the lab standing
|
|
||||||
// in for the builder — so it does here what the builder does: replace the artifact with the
|
|
||||||
// reference the machine actually holds. Without this the unresolved field travels to the
|
|
||||||
// machine, whose declaration language has no such field, and the whole declaration is refused.
|
|
||||||
//
|
|
||||||
// The repository is `mesh-runtime-<module>`, which is not a guess: it is what this repository's
|
|
||||||
// own `scripts/build-module-runtime.sh <module>` produces and what the scenarios stock by name.
|
|
||||||
if (typeof r.artifact === "string" && typeof r.image !== "string") {
|
|
||||||
const reference = referenceFor(held, `mesh-runtime-${name}`);
|
|
||||||
assert.ok(reference,
|
|
||||||
`${name} declares the "${r.artifact}" artifact and this scenario stocked no ` +
|
|
||||||
`mesh-runtime-${name}. The mesh would have to build it, and this bed does not build — ` +
|
|
||||||
`add it to the machine's images: in the scenario, or build it with ` +
|
|
||||||
`scripts/build-module-runtime.sh ${name}`);
|
|
||||||
r.image = reference;
|
|
||||||
delete r.artifact;
|
|
||||||
}
|
|
||||||
if (typeof r.image === "string") r.image = pinned(r.image);
|
|
||||||
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
|
|
||||||
}
|
|
||||||
const manifest = JSON.stringify(m);
|
|
||||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
function tokenFrom(said: string): string {
|
||||||
@@ -441,56 +410,8 @@ async function psMapOf(node: string): Promise<Map<string, string>> {
|
|||||||
return map;
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// ADR 0098: the internal authority makes its own root at first start and serves it; the proxy
|
||||||
* ADR 0066: the internal CA is initialised FROM AN OPERATOR'S ROOT — it does not mint its own.
|
// fetches it. No operator root is delivered — the mesh mints only the authority's password.
|
||||||
*
|
|
||||||
* So the bed has to be an operator. The material is made on the anchor with openssl and handed to
|
|
||||||
* the mesh through the real `secret accept` path, exactly as a person would: the mesh cannot invent
|
|
||||||
* a PEM, and the random 32 bytes it makes for an own-secret nobody supplied would leave step-ca
|
|
||||||
* crash-looping on a root key that is not a key.
|
|
||||||
*/
|
|
||||||
async function deliverCaRoot(): Promise<boolean> {
|
|
||||||
const made = await on(CONTROL, [
|
|
||||||
"set -e",
|
|
||||||
"mkdir -p /tmp/ca && cd /tmp/ca",
|
|
||||||
// No trailing newline on a password file: step-ca reads the file as the password itself.
|
|
||||||
"openssl rand -hex 16 | tr -d '\\n' > key-password",
|
|
||||||
"openssl ecparam -genkey -name prime256v1 -out root.unenc",
|
|
||||||
"openssl ec -in root.unenc -aes256 -passout file:key-password -out root.key",
|
|
||||||
"rm -f root.unenc",
|
|
||||||
"openssl req -x509 -new -key root.key -passin file:key-password -sha256 -days 3650" +
|
|
||||||
` -out root.crt -subj "/CN=Mesh Internal CA/O=Novox Mesh Lab"`,
|
|
||||||
// Readable by the control plane, which is not root. Its image is FROM scratch and runs as
|
|
||||||
// 65534, and `docker cp` keeps the ownership and mode a file had outside — openssl writes a
|
|
||||||
// private key 0600 root-owned, so the copy landed unreadable and `secret accept` failed with
|
|
||||||
// `open /ca-root-key: permission denied`. The CA then crash-looped on a root it never got.
|
|
||||||
// Chowning it inside the container is not available: there is no shell in there to do it with.
|
|
||||||
//
|
|
||||||
// Safe here and nowhere else: these three exist for the seconds between being written and
|
|
||||||
// being sealed to the machine, on a lab node, for a CA thrown away with the scenario.
|
|
||||||
"chmod 0644 /tmp/ca/root.crt /tmp/ca/root.key /tmp/ca/key-password",
|
|
||||||
"docker cp /tmp/ca/root.crt mesh-controller:/ca-root-cert",
|
|
||||||
"docker cp /tmp/ca/root.key mesh-controller:/ca-root-key",
|
|
||||||
"docker cp /tmp/ca/key-password mesh-controller:/ca-root-key-password",
|
|
||||||
].join("\n"), 180_000);
|
|
||||||
if (!made.ok) {
|
|
||||||
console.log(`CA ROOT NOT MADE on ${CONTROL}:\n${made.out.split("\n").slice(-8).join("\n")}`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
for (const [name, file] of [
|
|
||||||
["root-cert", "/ca-root-cert"],
|
|
||||||
["root-key", "/ca-root-key"],
|
|
||||||
["root-key-password", "/ca-root-key-password"],
|
|
||||||
] as const) {
|
|
||||||
try {
|
|
||||||
await mesh(`secret accept ${CONTROL} step-ca ${name} --from ${file}`);
|
|
||||||
} catch (err) {
|
|
||||||
console.log(`CA ROOT ACCEPT FAILED (${name}): ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** What a module's manifest says its route label is, or "" if it contributes no route. */
|
/** What a module's manifest says its route label is, or "" if it contributes no route. */
|
||||||
function routeLabelOf(name: string): string {
|
function routeLabelOf(name: string): string {
|
||||||
@@ -747,7 +668,6 @@ async function joinTheMesh(): Promise<void> {
|
|||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
if (skip) return;
|
if (skip) return;
|
||||||
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
@@ -880,6 +800,9 @@ test("the full mesh forms across the access point and both server sets converge"
|
|||||||
for (const { name } of mods) {
|
for (const { name } of mods) {
|
||||||
try {
|
try {
|
||||||
const broker = await ensureAdded(name);
|
const broker = await ensureAdded(name);
|
||||||
|
// Issued only when the module holds a broker account: an own secret the mesh mints
|
||||||
|
// (step-ca's password) is minted at resolve, and `module issue` refuses a module with no
|
||||||
|
// broker secret to deliver into (issue 078).
|
||||||
if (broker) await mesh(`module issue ${name} --node ${node}`);
|
if (broker) await mesh(`module issue ${name} --node ${node}`);
|
||||||
await mesh(`assign ${node} ${name}`);
|
await mesh(`assign ${node} ${name}`);
|
||||||
assigned[node]!.add(name);
|
assigned[node]!.add(name);
|
||||||
@@ -921,10 +844,6 @@ test("the full mesh forms across the access point and both server sets converge"
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ADR 0066: the CA's root, before the push that would otherwise deliver a random 32 bytes for it.
|
|
||||||
const caRootDelivered = assigned["novox"]!.has("step-ca") ? await deliverCaRoot() : false;
|
|
||||||
if (!caRootDelivered) console.log("ADR 0066: no operator root delivered; step-ca cannot initialise.");
|
|
||||||
|
|
||||||
// ONE push per node (workstations first — cheap — then the heavy service nodes).
|
// ONE push per node (workstations first — cheap — then the heavy service nodes).
|
||||||
const pushError: Record<string, string> = {};
|
const pushError: Record<string, string> = {};
|
||||||
for (const node of ["shanks", "g14", "novox", "ace"]) {
|
for (const node of ["shanks", "g14", "novox", "ace"]) {
|
||||||
@@ -1059,7 +978,6 @@ test("the full mesh forms across the access point and both server sets converge"
|
|||||||
? (await on("novox", `curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H 'Host: ${probe.name}' http://127.0.0.1/`)).out.trim()
|
? (await on("novox", `curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H 'Host: ${probe.name}' http://127.0.0.1/`)).out.trim()
|
||||||
: "";
|
: "";
|
||||||
adr.push(`\n proxy answers for ${probe?.name ?? "(nothing composed)"}: ${servedCode || "no answer"}`);
|
adr.push(`\n proxy answers for ${probe?.name ?? "(nothing composed)"}: ${servedCode || "no answer"}`);
|
||||||
adr.push(` operator root delivered to step-ca: ${caRootDelivered}`);
|
|
||||||
adr.push(` step-ca container: ${psMaps["novox"]?.get("step-ca") ?? "MISSING"}`);
|
adr.push(` step-ca container: ${psMaps["novox"]?.get("step-ca") ?? "MISSING"}`);
|
||||||
console.log(adr.join("\n"));
|
console.log(adr.join("\n"));
|
||||||
|
|
||||||
|
|||||||
@@ -36,19 +36,17 @@
|
|||||||
|
|
||||||
import { test, before, after } from "node:test";
|
import { test, before, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { existsSync, readFileSync } from "node:fs";
|
import { existsSync } from "node:fs";
|
||||||
import { dirname, resolve } from "node:path";
|
|
||||||
import { loadScenario } from "../../src/declaration/parse.ts";
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
const binary = hostBinaryPath();
|
const binary = hostBinaryPath();
|
||||||
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
||||||
const modulesEnv = process.env["MESH_LAB_MODULES"] ?? "";
|
|
||||||
|
|
||||||
const skip = !capability.usable
|
const skip = !capability.usable
|
||||||
? `lab not usable: ${capability.why}`
|
? `lab not usable: ${capability.why}`
|
||||||
@@ -56,15 +54,12 @@ const skip = !capability.usable
|
|||||||
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
||||||
: !bundle || !existsSync(bundle)
|
: !bundle || !existsSync(bundle)
|
||||||
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
||||||
: false;
|
: catalogueIsPresent();
|
||||||
|
|
||||||
const SCENARIO = "whole-mesh-novox";
|
const SCENARIO = "whole-mesh-novox";
|
||||||
const NODE = "novox";
|
const NODE = "novox";
|
||||||
|
|
||||||
/** Where the committed module.json files live: the mesh-catalog beside mesh-controller. */
|
|
||||||
const catalogDir = process.env["MESH_LAB_CATALOG"]
|
|
||||||
?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "")
|
|
||||||
?? resolve(process.cwd(), "..", "mesh-catalog", "modules");
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and
|
* The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and
|
||||||
@@ -73,6 +68,9 @@ const catalogDir = process.env["MESH_LAB_CATALOG"]
|
|||||||
*/
|
*/
|
||||||
const MODULES: { name: string; containers: string[]; node?: boolean }[] = [
|
const MODULES: { name: string; containers: string[]; node?: boolean }[] = [
|
||||||
{ name: "postgres", containers: ["mesh-store", "mesh-postgres"] },
|
{ name: "postgres", containers: ["mesh-store", "mesh-postgres"] },
|
||||||
|
// The vault, before everything that keeps a secret from it: redis, gitea, umami, influxdb,
|
||||||
|
// mailu require one (novox/hq ADRs 0085, 0094).
|
||||||
|
{ name: "mesh-vault", containers: ["mesh-vault"] },
|
||||||
{ name: "redis", containers: ["redis", "mesh-redis"] },
|
{ name: "redis", containers: ["redis", "mesh-redis"] },
|
||||||
{ name: "minio", containers: ["minio", "mesh-minio"] },
|
{ name: "minio", containers: ["minio", "mesh-minio"] },
|
||||||
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
|
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
|
||||||
@@ -229,38 +227,14 @@ function bundleFor(images: HeldImage[]): string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load a committed module.json, rewrite every container image to the scenario's pinned digest, and
|
* The catalogue's manifest as the lab runs it (harness): images pinned, artifacts resolved to the
|
||||||
* apply the host-port remaps. Returns the manifest as a string and whether it needs a broker account
|
* stocked images, the host-port remaps applied. Returns the manifest and whether it needs a broker
|
||||||
* (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules do not).
|
* account (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules
|
||||||
|
* do not).
|
||||||
*/
|
*/
|
||||||
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
function loadManifest(name: string): { manifest: string; broker: boolean } {
|
||||||
const path = resolve(catalogDir, name, "module.json");
|
const manifest = catalogueModule(name, held, { ports: REMAP[name] });
|
||||||
const m = JSON.parse(readFileSync(path, "utf8")) as {
|
return { manifest, broker: needsBrokerAccount(manifest) };
|
||||||
resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[];
|
|
||||||
build?: unknown;
|
|
||||||
};
|
|
||||||
const remap = REMAP[name] ?? {};
|
|
||||||
for (const r of m.resources ?? []) {
|
|
||||||
if (r.type !== "container") continue;
|
|
||||||
if (typeof r.image === "string") {
|
|
||||||
r.image = pinned(r.image);
|
|
||||||
} else if (typeof r.artifact === "string") {
|
|
||||||
// Since issue 060 a module's own runtime container names an artifact the mesh's builder
|
|
||||||
// would fill, not a placeholder image. This bed stocks the image instead of building, so
|
|
||||||
// map the artifact to the stocked `mesh-runtime-<module>` the machine holds — keyed on the
|
|
||||||
// MODULE name, not the container's (mailu's runtime container is `mesh-mailu`, its image is
|
|
||||||
// `mesh-runtime-mailu`). The placeholder digest is what `pinned` already resolves for a
|
|
||||||
// mesh-built repo, exactly as it did for the old `image` field.
|
|
||||||
r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`);
|
|
||||||
delete r.artifact;
|
|
||||||
}
|
|
||||||
if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p);
|
|
||||||
}
|
|
||||||
// The build section the mesh's builder would consume: dropped, because this bed stocks the image
|
|
||||||
// rather than building it. Harmless to leave (the push path never reads it), removed for clarity.
|
|
||||||
delete m.build;
|
|
||||||
const manifest = JSON.stringify(m);
|
|
||||||
return { manifest, broker: manifest.includes("MESH_BROKER_FILE") };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function tokenFrom(said: string): string {
|
function tokenFrom(said: string): string {
|
||||||
@@ -306,7 +280,6 @@ async function nodeState(node: string): Promise<NodeState> {
|
|||||||
|
|
||||||
before(async () => {
|
before(async () => {
|
||||||
if (skip) return;
|
if (skip) return;
|
||||||
assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`);
|
|
||||||
|
|
||||||
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
||||||
onProgress: (m) => console.log(`raise: ${m}`),
|
onProgress: (m) => console.log(`raise: ${m}`),
|
||||||
|
|||||||
@@ -104,12 +104,28 @@ test("every repository the receipt claims was built by the run", () => {
|
|||||||
MESH_LAB_HOST_BINARY: "/repo/host/mesh-host",
|
MESH_LAB_HOST_BINARY: "/repo/host/mesh-host",
|
||||||
MESH_LAB_MODULES: "/repo/control/examples/modules",
|
MESH_LAB_MODULES: "/repo/control/examples/modules",
|
||||||
MESH_LAB_BUILDER: "/repo/control/build/mesh-builder",
|
MESH_LAB_BUILDER: "/repo/control/build/mesh-builder",
|
||||||
|
MESH_LAB_CATALOG: "/repo/catalog/modules",
|
||||||
};
|
};
|
||||||
const built = new Set(planned(env).map((b) => b.in));
|
const built = new Set(planned(env).map((b) => b.in));
|
||||||
for (const [name, directory] of Object.entries(repositories(env))) {
|
for (const [name, directory] of Object.entries(repositories(env))) {
|
||||||
// mesh-lab is the exception, and it is not an omission: it is TypeScript run from source, so
|
// mesh-lab is the exception, and it is not an omission: it is TypeScript run from source, so
|
||||||
// the code under test *is* the code running. There is nothing to build and nothing to go stale.
|
// the code under test *is* the code running. There is nothing to build and nothing to go stale.
|
||||||
if (name === "mesh-lab") continue;
|
if (name === "mesh-lab") continue;
|
||||||
|
// mesh-catalog is the other exception, for the other reason: what a bed takes from it is a
|
||||||
|
// manifest, read from disk when the bed runs. There is nothing built from it that could go
|
||||||
|
// stale — and claiming it is the whole point, since a bed that carried its own copy of the
|
||||||
|
// manifest was proving the copy (novox/hq 04-ISSUES/073).
|
||||||
|
if (name === "mesh-catalog") continue;
|
||||||
assert.ok(built.has(directory), `${name} (${directory}) is claimed but never built`);
|
assert.ok(built.has(directory), `${name} (${directory}) is claimed but never built`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// The catalogue is claimed by the receipt, under either spelling the beds accept.
|
||||||
|
//
|
||||||
|
// A bed reads the manifest it installs from the catalogue checkout (novox/hq 04-ISSUES/073), so a
|
||||||
|
// receipt that names no catalogue commit cannot say whether a change there was ever proven.
|
||||||
|
test("the receipt claims the catalogue the beds read, however it was named", () => {
|
||||||
|
assert.equal(repositories({ MESH_LAB_CATALOG: "/repo/catalog/modules" })["mesh-catalog"], "/repo/catalog");
|
||||||
|
assert.equal(repositories({ MESH_LAB_CATALOG: "/repo/catalog" })["mesh-catalog"], "/repo/catalog");
|
||||||
|
assert.equal(repositories({})["mesh-catalog"], undefined);
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import { test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { ageOfImage, ageOfSource, isStale, plannedRuntimes, runtimesStockedBy, type Ask } from "../src/runtimes.ts";
|
||||||
|
|
||||||
|
// The module runtimes a run stocks are rebuilt by the run (novox/hq 04-ISSUES/075): what a bed's
|
||||||
|
// scenario stocks is found, compared against its source, and built where older.
|
||||||
|
|
||||||
|
function aLabWith(beds: Record<string, string>, scenarios: Record<string, string[]>): { root: string; done: () => void } {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), "mesh-lab-runtimes-"));
|
||||||
|
mkdirSync(join(root, "scenarios"));
|
||||||
|
mkdirSync(join(root, "test", "integration"), { recursive: true });
|
||||||
|
for (const [name, images] of Object.entries(scenarios)) {
|
||||||
|
writeFileSync(join(root, "scenarios", `${name}.yml`), `scenario: ${name}\nimages:\n${images.map((i) => ` - ${i}\n`).join("")}`);
|
||||||
|
}
|
||||||
|
for (const [file, scenario] of Object.entries(beds)) {
|
||||||
|
writeFileSync(join(root, "test", "integration", file), `const SCENARIO = "${scenario}";\n`);
|
||||||
|
}
|
||||||
|
return { root, done: () => rmSync(root, { recursive: true, force: true }) };
|
||||||
|
}
|
||||||
|
|
||||||
|
test("what a bed's scenario stocks is found, by module, once", () => {
|
||||||
|
const lab = aLabWith(
|
||||||
|
{ "a.test.ts": "one", "b.test.ts": "two", "c.test.ts": "one" },
|
||||||
|
{ one: ["mesh-controller:development", "mesh-runtime-gitlab:development", "postgres:16"],
|
||||||
|
two: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
|
||||||
|
try {
|
||||||
|
const found = runtimesStockedBy(["test/integration/a.test.ts", "test/integration/b.test.ts", "test/integration/c.test.ts"], lab.root);
|
||||||
|
assert.deepEqual(found, [
|
||||||
|
{ tag: "mesh-runtime-gitlab:development", module: "gitlab" },
|
||||||
|
// The audit logger's runtime is stocked under its slug, and the module is named for it.
|
||||||
|
{ tag: "mesh-runtime-audit:development", module: "audit-logger" },
|
||||||
|
]);
|
||||||
|
} finally { lab.done(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an image is stale when missing, older than its source, or when the source is uncommitted", () => {
|
||||||
|
assert.equal(isStale({ image: null, source: 0 }), true);
|
||||||
|
assert.equal(isStale({ image: 100, source: 200 }), true);
|
||||||
|
assert.equal(isStale({ image: 200, source: 100 }), false);
|
||||||
|
assert.equal(isStale({ image: 200, source: Infinity }), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the image's age comes from the store and the source's from the newest commit in any part", () => {
|
||||||
|
const answers: Ask = (command, args) => {
|
||||||
|
if (command === "docker") return { status: 0, stdout: "2026-09-21T12:00:00.000000000Z\n" };
|
||||||
|
if (args.includes("status")) return { status: 0, stdout: "" };
|
||||||
|
if (args.includes("modules/gitlab")) return { status: 0, stdout: "1000\n" };
|
||||||
|
return { status: 0, stdout: args[1] === "/tools" ? "3000\n" : "2000\n" };
|
||||||
|
};
|
||||||
|
assert.equal(ageOfImage("mesh-runtime-gitlab:development", answers), Date.parse("2026-09-21T12:00:00Z") / 1000);
|
||||||
|
assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], answers), 3000);
|
||||||
|
// No such image is null, not zero: "never built" and "built at the epoch" are different answers.
|
||||||
|
assert.equal(ageOfImage("mesh-runtime-nothing:development", () => ({ status: 1, stdout: "" })), null);
|
||||||
|
// Uncommitted changes anywhere in the source are newer than every commit.
|
||||||
|
const dirtyTools: Ask = (command, args) =>
|
||||||
|
args.includes("status") && args[1] === "/tools" ? { status: 0, stdout: " M src/x.ts\n" } : answers(command, args);
|
||||||
|
assert.equal(ageOfSource("/catalogue", "gitlab", ["/tools", "/sdk"], dirtyTools), Infinity);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("only a stale runtime is planned, built by the lab's own script under the tag the scenario stocks", () => {
|
||||||
|
const lab = aLabWith({ "a.test.ts": "one" },
|
||||||
|
{ one: ["mesh-runtime-gitlab:development", "mesh-runtime-audit:development"] });
|
||||||
|
try {
|
||||||
|
const answers: Ask = (command, args) => {
|
||||||
|
if (command === "docker") {
|
||||||
|
// gitlab's image is from yesterday; the audit logger has none.
|
||||||
|
return args.includes("mesh-runtime-gitlab:development")
|
||||||
|
? { status: 0, stdout: "2026-09-21T12:00:00Z\n" } : { status: 1, stdout: "" };
|
||||||
|
}
|
||||||
|
if (args.includes("status")) return { status: 0, stdout: "" };
|
||||||
|
return { status: 0, stdout: `${Date.parse("2026-09-20T00:00:00Z") / 1000}\n` };
|
||||||
|
};
|
||||||
|
const env = { MESH_LAB_CATALOG: "/catalogue/modules", MESH_TOOLS: "/tools", MESH_SDK: "/sdk" };
|
||||||
|
const builds = plannedRuntimes(["test/integration/a.test.ts"], env, lab.root, answers);
|
||||||
|
assert.equal(builds.length, 1);
|
||||||
|
assert.equal(builds[0]!.what, "runtime mesh-runtime-audit:development");
|
||||||
|
assert.equal(builds[0]!.argv[0], "scripts/build-module-runtime.sh");
|
||||||
|
assert.equal(builds[0]!.argv[1], "audit-logger");
|
||||||
|
assert.equal(builds[0]!.env?.["MESH_CATALOG"], "/catalogue");
|
||||||
|
assert.equal(builds[0]!.env?.["RUNTIME_TAG"], "mesh-runtime-audit:development");
|
||||||
|
// No catalogue named: nothing is planned, because no bed will read one either.
|
||||||
|
assert.deepEqual(plannedRuntimes(["test/integration/a.test.ts"], {}, lab.root, answers), []);
|
||||||
|
} finally { lab.done(); }
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user